From 747d6cbfb147c911d9dad1168a79b2314ed4564e Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Thu, 22 May 2025 19:40:08 +0200 Subject: [PATCH 1/6] refactor: Cleanup and minor improvements --- .../AlphaTabPlayground/media-sync-editor.tsx | 440 ++++++++++++++---- .../AlphaTabPlayground/styles.module.scss | 1 + 2 files changed, 345 insertions(+), 96 deletions(-) diff --git a/src/components/AlphaTabPlayground/media-sync-editor.tsx b/src/components/AlphaTabPlayground/media-sync-editor.tsx index cb4f68cec..ebf4c5152 100644 --- a/src/components/AlphaTabPlayground/media-sync-editor.tsx +++ b/src/components/AlphaTabPlayground/media-sync-editor.tsx @@ -16,7 +16,6 @@ export type MediaSyncEditorProps = { }; type MasterBarMarker = { - label: string; syncTime: number; synthTime: number; @@ -42,7 +41,7 @@ type SyncPointInfo = { // TODO: handle intermediate tempo changes and sync points -function ticksToMillis(tick: number, bpm: number): number { +function ticksToMilliseconds(tick: number, bpm: number): number { return (tick * 60000.0) / (bpm * 960); } @@ -67,65 +66,269 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo : [buffer.getChannelData(0), buffer.getChannelData(1)]; const sampleRate = audioContext.sampleRate; - const endTime = rawSamples[0].length / sampleRate; + const endTime = (rawSamples[0].length / sampleRate) * 1000; await audioContext.close(); - return { + const state: SyncPointInfo = { endTick: api.tickCache.masterBars.at(-1)!.end, - masterBarMarkers: buildMasterBarMarkers(api, createInitialSyncPoints), + masterBarMarkers: [], sampleRate, leftSamples: rawSamples[0], rightSamples: rawSamples[1], endTime }; -} -function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoints: boolean): MasterBarMarker[] { - const markers: MasterBarMarker[] = []; if (createInitialSyncPoints) { // create initial sync points for all tempo changes to ensure the song and the // backing track roughly align - let synthBpm = 0; + let synthBpm = api.tickCache!.masterBars[0].tempoChanges[0].tempo; let synthTimePosition = 0; let synthTickPosition = 0; + const syncPoints: MasterBarMarker[] = []; + + // first create all changes not respecting the song start and end const occurences = new Map(); for (const masterBar of api.tickCache!.masterBars) { const occurence = occurences.get(masterBar.masterBar.index) ?? 0; occurences.set(masterBar.masterBar.index, occurence + 1); + // we are guaranteed to have a tempo change per master bar indicating its own tempo + // (even though its not a change) for (const changes of masterBar.tempoChanges) { const absoluteTick = changes.tick; const tickOffset = absoluteTick - synthTickPosition; if (tickOffset > 0) { - const timeOffset = ticksToMillis(tickOffset, synthBpm); - + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); synthTickPosition = absoluteTick; synthTimePosition += timeOffset; } - if (changes.tempo !== synthBpm && changes.tick === masterBar.start) { - const syncPoint = new alphaTab.model.Automation(); - syncPoint.ratioPosition = 0; - syncPoint.type = alphaTab.model.AutomationType.SyncPoint; - syncPoint.syncPointValue = new alphaTab.model.SyncPointData(); - syncPoint.syncPointValue.barOccurence = occurence; - syncPoint.syncPointValue.millisecondOffset = synthTimePosition; - syncPoint.syncPointValue.modifiedTempo = changes.tempo; - masterBar.masterBar.addSyncPoint(syncPoint); + if (changes.tick === masterBar.start) { + const marker: MasterBarMarker = { + isStartMarker: masterBar.start === 0, + isEndMarker: false, + masterBarIndex: masterBar.masterBar.index, + occurence, + syncTime: synthTimePosition, + synthBpm, + synthTickDuration: masterBar.end - masterBar.start, + synthTime: synthTimePosition, + modifiedTempo: undefined + }; + + if (changes.tempo !== synthBpm || marker.isStartMarker) { + syncPoints.push(marker); + marker.modifiedTempo = changes.tempo; + } synthBpm = changes.tempo; + + state.masterBarMarkers.push(marker); + } else { + // TOOD: other tempo changes } } const tickOffset = masterBar.end - synthTickPosition; - const timeOffset = ticksToMillis(tickOffset, synthBpm); + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); synthTickPosition += tickOffset; synthTimePosition += timeOffset; } + + // end marker + const lastMasterBar = api.tickCache!.masterBars.at(-1)!; + state.masterBarMarkers.push({ + masterBarIndex: lastMasterBar.masterBar.index, + synthTickDuration: 0, + occurence: occurences.get(lastMasterBar.masterBar.index)!, + syncTime: synthTimePosition, + synthTime: synthTimePosition, + synthBpm, + modifiedTempo: synthBpm, + isStartMarker: false, + isEndMarker: true + }); + + // with the final durations known, we can "squeeze" together the song + // from start and end (keeping the relative positions) + // and the other bars will be adjusted accordingly + const [songStart, songEnd] = findAudioStartAndEnd(state); + + const synthDuration = synthTimePosition; + const realDuration = songEnd - songStart; + const scaleFactor = realDuration / synthDuration; + + // 1st Pass: shift all tempo change markers relatively and calculate BPM + let syncTime = songStart; + for (let i = 0; i < syncPoints.length; i++) { + const syncPoint = syncPoints[i]; + + syncPoint.syncTime = syncTime; + + if (i < 0) { + const previousMarker = syncPoints[i - 1]; + const synthDuration = syncPoint.synthTime - previousMarker.synthTime; + const syncedDuration = syncPoint.syncTime - previousMarker.syncTime; + const newBpm = (synthDuration / syncedDuration) * previousMarker.synthBpm; + previousMarker.modifiedTempo = newBpm; + } + + const ownStart = syncPoint.synthTime; + const nextStart = i < syncPoints.length - 1 ? syncPoints[i + 1].synthTime : ownStart; + + const oldDuration = nextStart - ownStart; + const newDuration = oldDuration * scaleFactor; + + syncTime += newDuration; + } + + // // 2nd Pass: adjust all in-between markers according to the new position + syncTime = songStart; + let syncedBpm = syncPoints[0].modifiedTempo!; + for (const marker of state.masterBarMarkers) { + marker.syncTime = syncTime; + + if (marker.modifiedTempo) { + syncedBpm = marker.modifiedTempo; + } + + syncTime += ticksToMilliseconds(marker.synthTickDuration, syncedBpm); + } + } else { + state.masterBarMarkers = buildMasterBarMarkers(api); + } + + return state; +} + +function resetSyncPoints(api: alphaTab.AlphaTabApi, state: SyncPointInfo): SyncPointInfo { + for (const b of api.score!.masterBars) { + b.syncPoints = undefined; + } + + return { + ...state, + // TODO: create initial ones + masterBarMarkers: buildMasterBarMarkers(api) + }; +} + +function findAudioStartAndEnd(state: SyncPointInfo): [number, number] { + // once we have 1s non-silent audio we consider it as start (or inverted for end) + const nonSilentSamplesThreshold = 1 * state.sampleRate; + // we accept 200ms of silence inbetween audible samples + const silentSamplesThreshold = 0.2 * state.sampleRate; + // there can always be a bit of a noise. we require some amplitude + // proper would be to consider the Frequency and calculate the + const nonSilentAmplitudeThreshold = 0.001; + + // we limit the search to 10s (from start/end), proper audio should not exceed this + const searchThreshold = Math.min(10 * state.sampleRate, state.leftSamples.length * 0.1); + + let songStart = searchThreshold; + let songEnd = state.leftSamples.length - searchThreshold; + + // find start offset + let sampleIndex = 0; + + let nonSilentSamplesInSection = 0; + let silentSamplesInSequence = 0; + let sectionStart = 0; + + while (sampleIndex < songStart) { + if ( + Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || + Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold + ) { + // the first audible sample marks the potential start + if (nonSilentSamplesInSection === 0) { + sectionStart = sampleIndex; + } + nonSilentSamplesInSection++; + silentSamplesInSequence = 0; + } else { + silentSamplesInSequence++; + } + + // found more than X-samples silent, no start until here + if (silentSamplesInSequence > silentSamplesThreshold) { + // reset and start searching agian + sectionStart = sampleIndex + 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + } + // found enough samples since section start which are audible, should be good + else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { + songStart = sectionStart; + break; + } + + sampleIndex++; + } + + // and same from the back + sampleIndex = state.leftSamples.length - 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + sectionStart = sampleIndex; + + while (sampleIndex >= songEnd) { + if ( + Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || + Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold + ) { + if (nonSilentSamplesInSection === 0) { + sectionStart = sampleIndex; + } + nonSilentSamplesInSection++; + silentSamplesInSequence = 0; + } else { + silentSamplesInSequence++; + } + + if (silentSamplesInSequence > silentSamplesThreshold) { + sectionStart = sampleIndex - 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + } else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { + songEnd = sectionStart; + break; + } + + sampleIndex--; } + return [(songStart / state.sampleRate) * 1000, (songEnd / state.sampleRate) * 1000]; +} + +function cropToAudio(state: SyncPointInfo): SyncPointInfo { + const [songStart, songEnd] = findAudioStartAndEnd(state); + const newState: SyncPointInfo = { + ...state, + masterBarMarkers: [...state.masterBarMarkers] + }; + // move first marker + newState.masterBarMarkers[0] = { + ...newState.masterBarMarkers[0], + syncTime: songStart + }; + updateSyncPointsAfterModification(0, newState, false, true); + + // move last marker + newState.masterBarMarkers[newState.masterBarMarkers.length - 1] = { + ...newState.masterBarMarkers[newState.masterBarMarkers.length - 1], + syncTime: songEnd + }; + updateSyncPointsAfterModification(newState.masterBarMarkers.length - 1, newState, false, true); + + return newState; +} + +function buildMasterBarMarkers(api: alphaTab.AlphaTabApi): MasterBarMarker[] { + const markers: MasterBarMarker[] = []; + const occurences = new Map(); let syncBpm = api.score!.tempo; let syncLastTick = 0; @@ -136,11 +339,12 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint let synthTickPosition = 0; for (const masterBar of api.tickCache!.masterBars) { - const occurence = occurences.get(masterBar.masterBar.index) ?? 1; + const occurence = occurences.get(masterBar.masterBar.index) ?? 0; occurences.set(masterBar.masterBar.index, occurence + 1); - const occurenceLabel = occurence > 1 ? ` (${occurence})` : ''; - const startSyncPoint = masterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 0); + const startSyncPoint = masterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 0 && + m.syncPointValue!.barOccurence === occurence + ); let syncedStartTime: number; if (startSyncPoint) { @@ -150,17 +354,16 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint syncLastTick = masterBar.start; } else { const tickOffset = masterBar.start - syncLastTick; - syncedStartTime = syncLastMillisecondOffset + ticksToMillis(tickOffset, syncBpm); + syncedStartTime = syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm); } - const isStartMarker = masterBar.masterBar.index === 0 && occurence === 1; + const isStartMarker = masterBar.masterBar.index === 0 && occurence === 0; const newMarker: MasterBarMarker = { - label: isStartMarker ? 'Start' : `${masterBar.masterBar.index + 1}${occurenceLabel}`, masterBarIndex: masterBar.masterBar.index, synthTickDuration: masterBar.end - masterBar.start, occurence: occurence, - syncTime: syncedStartTime / 1000, - synthTime: synthTimePosition / 1000, + syncTime: syncedStartTime, + synthTime: synthTimePosition, synthBpm: masterBar.tempoChanges.length > 0 ? masterBar.tempoChanges[0].tempo : synthBpm, modifiedTempo: startSyncPoint?.syncPointValue?.modifiedTempo, isStartMarker, @@ -172,7 +375,7 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint const absoluteTick = changes.tick; const tickOffset = absoluteTick - synthTickPosition; if (tickOffset > 0) { - const timeOffset = ticksToMillis(tickOffset, synthBpm); + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); synthTickPosition = absoluteTick; synthTimePosition += timeOffset; @@ -182,7 +385,7 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint } const tickOffset = masterBar.end - synthTickPosition; - const timeOffset = ticksToMillis(tickOffset, synthBpm); + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); synthTickPosition += tickOffset; synthTimePosition += timeOffset; } @@ -194,15 +397,14 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint const tickOffset = lastMasterBar.end - syncLastTick; const endSyncPointTime = endSyncPoint ? endSyncPoint.syncPointValue!.millisecondOffset - : syncLastMillisecondOffset + ticksToMillis(tickOffset, syncBpm); + : syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm); markers.push({ - label: 'End', masterBarIndex: lastMasterBar.masterBar.index, synthTickDuration: 0, occurence: occurences.get(lastMasterBar.masterBar.index)!, - syncTime: endSyncPointTime / 1000, - synthTime: synthTimePosition / 1000, + syncTime: endSyncPointTime, + synthTime: synthTimePosition, synthBpm, modifiedTempo: endSyncPoint?.syncPointValue?.modifiedTempo ?? synthBpm, isStartMarker: false, @@ -212,7 +414,7 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoint return markers; } -const pixelPerSeconds = 100; +const pixelPerMilliseconds = 100 / 1000 /* 100px per 1000ms */; const leftPadding = 15; const barNumberHeight = 20; const arrowHeight = 20; @@ -227,8 +429,8 @@ const dragThreshold = 5; const scrollThresholdPercent = 0.2; function timePositionToX(timePosition: number, zoom: number): number { - const zoomedPixelPerSecond = pixelPerSeconds * zoom; - return timePosition * zoomedPixelPerSecond + leftPadding; + const zoomedPixelPerMilliseconds = pixelPerMilliseconds * zoom; + return timePosition * zoomedPixelPerMilliseconds + leftPadding; } type MarkerDragInfo = { @@ -255,7 +457,12 @@ function computeMarkerInlineStyle( }; } -function updateSyncPointsAfterModification(modifiedIndex: number, s: SyncPointInfo, isDelete: boolean) { +function updateSyncPointsAfterModification( + modifiedIndex: number, + s: SyncPointInfo, + isDelete: boolean, + cloneMarkers: boolean +) { // find previous and next sync point (or start/end of the song) let startIndexForUpdate = Math.max(0, modifiedIndex - 1); while (startIndexForUpdate > 0 && !s.masterBarMarkers[startIndexForUpdate].modifiedTempo) { @@ -274,7 +481,9 @@ function updateSyncPointsAfterModification(modifiedIndex: number, s: SyncPointIn // update from previous to current if (startIndexForUpdate < modifiedIndex) { - const previousMarker = { ...s.masterBarMarkers[startIndexForUpdate] }; + const previousMarker = cloneMarkers + ? { ...s.masterBarMarkers[startIndexForUpdate] } + : s.masterBarMarkers[startIndexForUpdate]; s.masterBarMarkers[startIndexForUpdate] = previousMarker; const synthDuration = modifiedMarker.synthTime - previousMarker.synthTime; const syncedDuration = modifiedMarker.syncTime - previousMarker.syncTime; @@ -283,11 +492,11 @@ function updateSyncPointsAfterModification(modifiedIndex: number, s: SyncPointIn let syncedTimePosition = previousMarker.syncTime; for (let i = startIndexForUpdate; i < modifiedIndex; i++) { - const marker = { ...s.masterBarMarkers[i] }; + const marker = cloneMarkers ? { ...s.masterBarMarkers[i] } : s.masterBarMarkers[i]; s.masterBarMarkers[i] = marker; marker.syncTime = syncedTimePosition; - syncedTimePosition += ticksToMillis(marker.synthTickDuration, newBpmBefore) / 1000; + syncedTimePosition += ticksToMilliseconds(marker.synthTickDuration, newBpmBefore); } } @@ -299,13 +508,13 @@ function updateSyncPointsAfterModification(modifiedIndex: number, s: SyncPointIn modifiedMarker.modifiedTempo = newBpmAfter; let syncedTimePosition = - modifiedMarker.syncTime + ticksToMillis(modifiedMarker.synthTickDuration, newBpmAfter) / 1000; + modifiedMarker.syncTime + ticksToMilliseconds(modifiedMarker.synthTickDuration, newBpmAfter); for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) { - const marker = { ...s.masterBarMarkers[i] }; + const marker = cloneMarkers ? { ...s.masterBarMarkers[i] } : s.masterBarMarkers[i]; s.masterBarMarkers[i] = marker; marker.syncTime = syncedTimePosition; - syncedTimePosition += ticksToMillis(marker.synthTickDuration, newBpmAfter) / 1000; + syncedTimePosition += ticksToMilliseconds(marker.synthTickDuration, newBpmAfter); } } } @@ -370,7 +579,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) const scrollOffset = syncArea.current.scrollLeft; // is out of screen? - if (xPos < scrollOffset + threshold || (xPos - scrollOffset) > (canvasWidth - threshold)) { + if (xPos < scrollOffset + threshold || xPos - scrollOffset > canvasWidth - threshold) { syncArea.current.scrollTo({ left: xPos - canvasWidth / 2, behavior: 'smooth' @@ -381,7 +590,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) useEffect(() => { const updateWaveFormCursor = () => { - setPlaybackTime(audioElement!.currentTime); + setPlaybackTime(audioElement!.currentTime * 1000); }; let timeUpdate: number = 0; @@ -393,7 +602,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) audioElement.addEventListener('seeked', updateWaveFormCursor); timeUpdate = window.setInterval(() => { if (audioElement) { - setPlaybackTime(audioElement.currentTime); + setPlaybackTime(audioElement.currentTime * 1000); } }, 50); updateWaveFormCursor(); @@ -436,8 +645,8 @@ export const MediaSyncEditor: React.FC = ({ api, score }) automation.type = alphaTab.model.AutomationType.SyncPoint; automation.syncPointValue = new alphaTab.model.SyncPointData(); automation.syncPointValue.modifiedTempo = m.modifiedTempo; - automation.syncPointValue.millisecondOffset = m.syncTime * 1000; - automation.syncPointValue.barOccurence = m.occurence - 1; + automation.syncPointValue.millisecondOffset = m.syncTime; + automation.syncPointValue.barOccurence = m.occurence; syncPoints.push(automation); } } @@ -507,9 +716,9 @@ export const MediaSyncEditor: React.FC = ({ api, score }) const newS = { ...s, masterBarMarkers: [...s.masterBarMarkers] }; if (marker.modifiedTempo) { newS.masterBarMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; - updateSyncPointsAfterModification(markerIndex, newS, true); + updateSyncPointsAfterModification(markerIndex, newS, true, true); } else { - updateSyncPointsAfterModification(markerIndex, newS, false); + updateSyncPointsAfterModification(markerIndex, newS, false, true); } return newS; @@ -533,8 +742,8 @@ export const MediaSyncEditor: React.FC = ({ api, score }) const markerIndex = s.masterBarMarkers.findIndex(m => m === draggingMarker); - const zoomedPixelPerSecond = pixelPerSeconds * zoom; - const deltaTime = deltaX / zoomedPixelPerSecond; + const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; + const deltaTime = deltaX / zoomedPixelPerMillisecond; const newTimePosition = draggingMarker.syncTime + deltaTime; @@ -546,7 +755,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) syncTime: Math.max(0, newTimePosition) }; - updateSyncPointsAfterModification(markerIndex, newS, false); + updateSyncPointsAfterModification(markerIndex, newS, false, true); return newS; }); setDraggingMarker(null); @@ -658,11 +867,11 @@ export const MediaSyncEditor: React.FC = ({ api, score }) ctx.beginPath(); - const startX = syncArea.current!.scrollLeft; - const endX = startX + can.width; + const startX = Math.max(syncArea.current!.scrollLeft - leftPadding, 0); + const endX = startX + can.width + leftPadding; - const zoomedPixelPerSecond = pixelPerSeconds * zoom; - const samplesPerPixel = syncPointInfo.sampleRate / zoomedPixelPerSecond; + const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; + const samplesPerPixel = syncPointInfo.sampleRate / (zoomedPixelPerMillisecond * 1000); for (let x = startX; x < endX; x += barWidth) { const startSample = (x * samplesPerPixel) | 0; @@ -671,11 +880,8 @@ export const MediaSyncEditor: React.FC = ({ api, score }) let maxTop = 0; let maxBottom = 0; for (let sample = startSample; sample <= endSample; sample++) { - // TODO: a logarithmic scale would be better here to scale 0-1 better as visible waveform - // for now we multiply it for a good scale (unlikely we have a sound with 1 which is very loud) - const visibilityFactor = 5; - const magnitudeTop = Math.min(Math.abs(syncPointInfo.leftSamples[sample] * visibilityFactor || 0), 1); - const magnitudeBottom = Math.min(Math.abs(syncPointInfo.rightSamples[sample] * visibilityFactor || 0), 1); + const magnitudeTop = Math.abs(syncPointInfo.leftSamples[sample] || 0); + const magnitudeBottom = Math.abs(syncPointInfo.rightSamples[sample] || 0); if (magnitudeTop > maxTop) { maxTop = magnitudeTop; } @@ -684,10 +890,10 @@ export const MediaSyncEditor: React.FC = ({ api, score }) } } - const topBarHeight = Math.round(maxTop * halfHeight); - const bottomBarHeight = Math.round(maxBottom * halfHeight); + const topBarHeight = Math.min(halfHeight, Math.round(maxTop * halfHeight)); + const bottomBarHeight = Math.min(halfHeight, Math.round(maxBottom * halfHeight)); const barHeight = topBarHeight + bottomBarHeight || 1; - ctx.rect(x, waveFormY + (halfHeight - topBarHeight), barWidth, barHeight); + ctx.rect(x + leftPadding, waveFormY + (halfHeight - topBarHeight), barWidth, barHeight); } ctx.fillStyle = waveFormColor; @@ -702,31 +908,35 @@ export const MediaSyncEditor: React.FC = ({ api, score }) ctx.textBaseline = 'bottom'; const timeAxisY = waveFormY + 2 * halfHeight; - const leftTime = Math.floor((startX - leftPadding) / zoomedPixelPerSecond); - const rightTime = Math.ceil(endX / zoomedPixelPerSecond); + const leftTimeSecond = Math.floor((startX - leftPadding) / zoomedPixelPerMillisecond / 1000); + const rightTimeSecond = Math.ceil(endX / zoomedPixelPerMillisecond / 1000); + + const leftTime = leftTimeSecond * 1000; + const rightTime = rightTimeSecond * 1000; let time = leftTime; while (time <= rightTime) { const timeX = timePositionToX(time, zoom); ctx.fillRect(timeX, timeAxisY, 1, timeAxisHeight); - const minutes = Math.floor(time / 60); - const seconds = Math.floor(time - minutes * 60); + const totalSeconds = Math.abs(time / 1000); + + const minutes = Math.floor(totalSeconds / 60); + const seconds = Math.floor(totalSeconds - minutes * 60); + const sign = time < 0 ? '-' : ''; const minutesText = minutes.toString().padStart(2, '0'); const secondsText = seconds.toString().padStart(2, '0'); - ctx.fillText(`${minutesText}:${secondsText}`, timeX + 3, timeAxisY + timeAxisHeight); + ctx.fillText(`${sign}${minutesText}:${secondsText}`, timeX + 3, timeAxisY + timeAxisHeight); - const nextSecond = time + 1; + const nextSecond = time + 1000; while (time < nextSecond) { const subSecondX = timePositionToX(time, zoom); ctx.fillRect(subSecondX, timeAxisY, 1, timeAxiSubSecondTickHeight); - time += 0.1; + time += 100; } - - time = Math.floor(time + 0.5); } ctx.restore(); @@ -734,8 +944,12 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }; useEffect(() => { + if (waveFormCanvas.current) { + waveFormCanvas.current.width = canvasSize[0]; + waveFormCanvas.current.height = canvasSize[1]; + } drawWaveform(); - }, [canvasSize, virtualWidth]); + }, [waveFormCanvas, canvasSize]); useResizeObserver(syncArea, entry => { setCanvasSize(s => [entry.contentRect.width, entry.contentRect.height]); @@ -743,7 +957,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) useEffect(() => { if (syncPointInfo) { - setVirtualWidth(s => pixelPerSeconds * syncPointInfo.endTime * zoom); + setVirtualWidth(s => pixelPerMilliseconds * syncPointInfo.endTime * zoom); } drawWaveform(); }, [markerCanvas, syncPointInfo, zoom]); @@ -772,6 +986,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) score.backingTrack.rawAudioFile = new Uint8Array(e.target!.result as ArrayBuffer); // create a fresh set of sync points upon load (start->end) + setApplySyncPoints(true); setCreateInitialSyncPoints(true); }; reader.readAsArrayBuffer(input.files[0]); @@ -782,6 +997,36 @@ export const MediaSyncEditor: React.FC = ({ api, score }) document.body.removeChild(input); }; + const onResetSyncPoints = () => { + setApplySyncPoints(true); + setStoreToUndo(true); + setSyncPointInfo(s => { + return s ? resetSyncPoints(api, s) : s; + }); + }; + + const onCropToAudio = () => { + setApplySyncPoints(true); + setStoreToUndo(true); + setSyncPointInfo(s => { + return s ? cropToAudio(s) : s; + }); + }; + + function buildMarkerLabel(m: MasterBarMarker): React.ReactNode { + if (m.isStartMarker) { + return 'Start'; + } + if (m.isEndMarker) { + return 'End'; + } + + if (m.occurence > 0) { + return `${m.masterBarIndex + 1} (${m.occurence + 1})`; + } + return `${m.masterBarIndex + 1}`; + } + return (
drawWaveform()}>
- +
{syncPointInfo.masterBarMarkers.map(m => (
toggleMarker(m, e)} onMouseDown={e => { startMarkerDrag(m, e); }}> -
{m.label}
+
{buildMarkerLabel(m)}
{!m.isEndMarker && m.modifiedTempo && ( diff --git a/src/components/AlphaTabPlayground/styles.module.scss b/src/components/AlphaTabPlayground/styles.module.scss index 386260467..571bb6063 100644 --- a/src/components/AlphaTabPlayground/styles.module.scss +++ b/src/components/AlphaTabPlayground/styles.module.scss @@ -452,6 +452,7 @@ top: 0; bottom: 20px; transform: translateX(-50%); + user-select: none; display: flex; flex-direction: column; From 0a13c7f0efbe124be5039e71fc19cd42b95172fb Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Thu, 22 May 2025 21:19:46 +0200 Subject: [PATCH 2/6] feat: handle intermediate sync points --- .../AlphaTabPlayground/media-sync-editor.tsx | 468 ++++++++++++------ .../AlphaTabPlayground/styles.module.scss | 33 +- 2 files changed, 332 insertions(+), 169 deletions(-) diff --git a/src/components/AlphaTabPlayground/media-sync-editor.tsx b/src/components/AlphaTabPlayground/media-sync-editor.tsx index ebf4c5152..149965b6a 100644 --- a/src/components/AlphaTabPlayground/media-sync-editor.tsx +++ b/src/components/AlphaTabPlayground/media-sync-editor.tsx @@ -10,24 +10,33 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import useResizeObserver from '@react-hook/resize-observer'; import { useAlphaTabEvent } from '@site/src/hooks'; +// TODO: cleanup the code (splitup helpers and components properly) + export type MediaSyncEditorProps = { api: alphaTab.AlphaTabApi; score: alphaTab.model.Score; }; -type MasterBarMarker = { +enum SyncPointMarkerType { + StartMarker = 0, + EndMarker = 1, + MasterBar = 2, + Intermediate = 3 +} + +type SyncPointMarker = { syncTime: number; synthTime: number; synthBpm: number; - synthTickDuration: number; + synthTick: number; masterBarIndex: number; + ratioPosition: number; occurence: number; modifiedTempo?: number; - isStartMarker: boolean; - isEndMarker: boolean; + markerType: SyncPointMarkerType; }; type SyncPointInfo = { @@ -36,16 +45,17 @@ type SyncPointInfo = { sampleRate: number; leftSamples: Float32Array; rightSamples: Float32Array; - masterBarMarkers: MasterBarMarker[]; + syncPointMarkers: SyncPointMarker[]; }; -// TODO: handle intermediate tempo changes and sync points - function ticksToMilliseconds(tick: number, bpm: number): number { return (tick * 60000.0) / (bpm * 960); } -async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPoints: boolean): Promise { +async function buildSyncPointInfoFromApi( + api: alphaTab.AlphaTabApi, + createInitialSyncPoints: boolean +): Promise { const tickCache = api.tickCache; if (!tickCache || !api.score?.backingTrack?.rawAudioFile) { return { @@ -54,7 +64,7 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo sampleRate: 0, leftSamples: new Float32Array(0), rightSamples: new Float32Array(0), - masterBarMarkers: [] + syncPointMarkers: [] }; } @@ -72,7 +82,7 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo const state: SyncPointInfo = { endTick: api.tickCache.masterBars.at(-1)!.end, - masterBarMarkers: [], + syncPointMarkers: [], sampleRate, leftSamples: rawSamples[0], rightSamples: rawSamples[1], @@ -86,7 +96,7 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo let synthTimePosition = 0; let synthTickPosition = 0; - const syncPoints: MasterBarMarker[] = []; + const syncPoints: SyncPointMarker[] = []; // first create all changes not respecting the song start and end const occurences = new Map(); @@ -105,30 +115,34 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo synthTimePosition += timeOffset; } - if (changes.tick === masterBar.start) { - const marker: MasterBarMarker = { - isStartMarker: masterBar.start === 0, - isEndMarker: false, - masterBarIndex: masterBar.masterBar.index, - occurence, - syncTime: synthTimePosition, - synthBpm, - synthTickDuration: masterBar.end - masterBar.start, - synthTime: synthTimePosition, - modifiedTempo: undefined - }; - - if (changes.tempo !== synthBpm || marker.isStartMarker) { - syncPoints.push(marker); - marker.modifiedTempo = changes.tempo; - } + const marker: SyncPointMarker = { + markerType: SyncPointMarkerType.MasterBar, + masterBarIndex: masterBar.masterBar.index, + occurence, + syncTime: synthTimePosition, + synthBpm, + synthTime: synthTimePosition, + modifiedTempo: undefined, + ratioPosition: 0, + synthTick: synthTickPosition + }; - synthBpm = changes.tempo; + if (masterBar.start === 0) { + marker.markerType = SyncPointMarkerType.StartMarker; + } else if (changes.tick > masterBar.start) { + marker.markerType = SyncPointMarkerType.Intermediate; + const duration = masterBar.start - masterBar.end; + marker.ratioPosition = changes.tick / duration; + } - state.masterBarMarkers.push(marker); - } else { - // TOOD: other tempo changes + if (changes.tempo !== synthBpm || marker.markerType === SyncPointMarkerType.StartMarker) { + syncPoints.push(marker); + marker.modifiedTempo = changes.tempo; } + + synthBpm = changes.tempo; + + state.syncPointMarkers.push(marker); } const tickOffset = masterBar.end - synthTickPosition; @@ -139,16 +153,16 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo // end marker const lastMasterBar = api.tickCache!.masterBars.at(-1)!; - state.masterBarMarkers.push({ + state.syncPointMarkers.push({ masterBarIndex: lastMasterBar.masterBar.index, - synthTickDuration: 0, occurence: occurences.get(lastMasterBar.masterBar.index)!, syncTime: synthTimePosition, synthTime: synthTimePosition, synthBpm, modifiedTempo: synthBpm, - isStartMarker: false, - isEndMarker: true + markerType: SyncPointMarkerType.EndMarker, + ratioPosition: 1, + synthTick: synthTickPosition }); // with the final durations known, we can "squeeze" together the song @@ -184,20 +198,24 @@ async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPo syncTime += newDuration; } - // // 2nd Pass: adjust all in-between markers according to the new position + // 2nd Pass: adjust all in-between markers according to the new position syncTime = songStart; let syncedBpm = syncPoints[0].modifiedTempo!; - for (const marker of state.masterBarMarkers) { + for (let i = 0; i < state.syncPointMarkers.length; i++) { + const marker = state.syncPointMarkers[i]; marker.syncTime = syncTime; if (marker.modifiedTempo) { syncedBpm = marker.modifiedTempo; } - syncTime += ticksToMilliseconds(marker.synthTickDuration, syncedBpm); + if (i < state.syncPointMarkers.length - 1) { + const tickDiff = state.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncTime += ticksToMilliseconds(tickDiff, syncedBpm); + } } } else { - state.masterBarMarkers = buildMasterBarMarkers(api); + state.syncPointMarkers = buildSyncPointMarkers(api); } return state; @@ -210,8 +228,7 @@ function resetSyncPoints(api: alphaTab.AlphaTabApi, state: SyncPointInfo): SyncP return { ...state, - // TODO: create initial ones - masterBarMarkers: buildMasterBarMarkers(api) + syncPointMarkers: buildSyncPointMarkers(api) }; } @@ -307,27 +324,27 @@ function cropToAudio(state: SyncPointInfo): SyncPointInfo { const [songStart, songEnd] = findAudioStartAndEnd(state); const newState: SyncPointInfo = { ...state, - masterBarMarkers: [...state.masterBarMarkers] + syncPointMarkers: [...state.syncPointMarkers] }; // move first marker - newState.masterBarMarkers[0] = { - ...newState.masterBarMarkers[0], + newState.syncPointMarkers[0] = { + ...newState.syncPointMarkers[0], syncTime: songStart }; updateSyncPointsAfterModification(0, newState, false, true); // move last marker - newState.masterBarMarkers[newState.masterBarMarkers.length - 1] = { - ...newState.masterBarMarkers[newState.masterBarMarkers.length - 1], + newState.syncPointMarkers[newState.syncPointMarkers.length - 1] = { + ...newState.syncPointMarkers[newState.syncPointMarkers.length - 1], syncTime: songEnd }; - updateSyncPointsAfterModification(newState.masterBarMarkers.length - 1, newState, false, true); + updateSyncPointsAfterModification(newState.syncPointMarkers.length - 1, newState, false, true); return newState; } -function buildMasterBarMarkers(api: alphaTab.AlphaTabApi): MasterBarMarker[] { - const markers: MasterBarMarker[] = []; +function buildSyncPointMarkers(api: alphaTab.AlphaTabApi): SyncPointMarker[] { + const markers: SyncPointMarker[] = []; const occurences = new Map(); let syncBpm = api.score!.tempo; @@ -342,56 +359,131 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi): MasterBarMarker[] { const occurence = occurences.get(masterBar.masterBar.index) ?? 0; occurences.set(masterBar.masterBar.index, occurence + 1); - const startSyncPoint = masterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 0 && - m.syncPointValue!.barOccurence === occurence - ); + const duration = masterBar.end - masterBar.start; - let syncedStartTime: number; - if (startSyncPoint) { - syncedStartTime = startSyncPoint.syncPointValue!.millisecondOffset; - syncBpm = startSyncPoint.syncPointValue!.modifiedTempo; - syncLastMillisecondOffset = syncedStartTime; - syncLastTick = masterBar.start; - } else { - const tickOffset = masterBar.start - syncLastTick; - syncedStartTime = syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm); - } + if (masterBar.masterBar.syncPoints) { + // if we have sync points we have to correctly walk through the points and tempo changes + // and place the markers accordingly - const isStartMarker = masterBar.masterBar.index === 0 && occurence === 0; - const newMarker: MasterBarMarker = { - masterBarIndex: masterBar.masterBar.index, - synthTickDuration: masterBar.end - masterBar.start, - occurence: occurence, - syncTime: syncedStartTime, - synthTime: synthTimePosition, - synthBpm: masterBar.tempoChanges.length > 0 ? masterBar.tempoChanges[0].tempo : synthBpm, - modifiedTempo: startSyncPoint?.syncPointValue?.modifiedTempo, - isStartMarker, - isEndMarker: false - }; - markers.push(newMarker); + // TODO: create placeholder markers matching the time signature and relative offsets. + + let tempoChangeIndex = 0; + for (const syncPoint of masterBar.masterBar.syncPoints) { + if (syncPoint.syncPointValue!.barOccurence !== occurence) { + continue; + } - for (const changes of masterBar.tempoChanges) { - const absoluteTick = changes.tick; - const tickOffset = absoluteTick - synthTickPosition; - if (tickOffset > 0) { - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + const syncPointTick = masterBar.start + syncPoint.ratioPosition * duration; + + // first process all tempo change until this sync point + while ( + tempoChangeIndex < masterBar.tempoChanges.length && + masterBar.tempoChanges[tempoChangeIndex].tick <= syncPointTick + ) { + const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; + const absoluteTick = tempoChange.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } - synthTickPosition = absoluteTick; - synthTimePosition += timeOffset; + synthBpm = tempoChange.tempo; + tempoChangeIndex++; + } + + // process time until sync point + const tickOffset = syncPointTick - synthTickPosition; + if (tickOffset > 0) { + synthTickPosition = syncPointTick; + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTimePosition += timeOffset; + } + + // create sync point marker + const newMarker: SyncPointMarker = { + masterBarIndex: masterBar.masterBar.index, + occurence: occurence, + syncTime: syncPoint.syncPointValue!.millisecondOffset, + synthTime: synthTimePosition, + synthBpm: masterBar.tempoChanges[0].tempo, + modifiedTempo: syncPoint!.syncPointValue!.modifiedTempo, + markerType: + syncPoint.ratioPosition === 0 + ? SyncPointMarkerType.MasterBar + : SyncPointMarkerType.Intermediate, + ratioPosition: syncPoint.ratioPosition, + synthTick: synthTickPosition + }; + if (syncPointTick === 0) { + newMarker.markerType = SyncPointMarkerType.StartMarker; + } + markers.push(newMarker); + + // remember values for artificially generated markers + syncBpm = syncPoint.syncPointValue!.modifiedTempo; + syncLastMillisecondOffset = syncPoint.syncPointValue!.millisecondOffset; + syncLastTick = masterBar.start; } - synthBpm = changes.tempo; - } + // process remaining tempo changes after all sync points + while (tempoChangeIndex < masterBar.tempoChanges.length) { + const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; + const absoluteTick = tempoChange.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } - const tickOffset = masterBar.end - synthTickPosition; - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition += tickOffset; - synthTimePosition += timeOffset; + synthBpm = tempoChange.tempo; + tempoChangeIndex++; + } + } else { + // TODO: Create intermediate markers matching time signature + + // if there are no sync points, we create a main masterbar sync point marker at start + let tickOffset = masterBar.start - syncLastTick; + + const newMarker: SyncPointMarker = { + masterBarIndex: masterBar.masterBar.index, + occurence: occurence, + syncTime: syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm), + synthTime: synthTimePosition, + synthTick: synthTickPosition, + synthBpm: masterBar.tempoChanges[0].tempo, + modifiedTempo: undefined, + markerType: masterBar.start === 0 ? SyncPointMarkerType.StartMarker : SyncPointMarkerType.MasterBar, + ratioPosition: 0 + }; + markers.push(newMarker); + + // and then we walk through the tempo changes + for (const changes of masterBar.tempoChanges) { + const absoluteTick = changes.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + synthBpm = changes.tempo; + } + + // don't forget the part after the last tempo change + tickOffset = masterBar.end - synthTickPosition; + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition += tickOffset; + synthTimePosition += timeOffset; + } } + // at the very end we create the end marker const lastMasterBar = api.tickCache!.masterBars.at(-1)!; - const endSyncPoint = lastMasterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 1); const tickOffset = lastMasterBar.end - syncLastTick; @@ -401,14 +493,14 @@ function buildMasterBarMarkers(api: alphaTab.AlphaTabApi): MasterBarMarker[] { markers.push({ masterBarIndex: lastMasterBar.masterBar.index, - synthTickDuration: 0, occurence: occurences.get(lastMasterBar.masterBar.index)!, syncTime: endSyncPointTime, synthTime: synthTimePosition, synthBpm, modifiedTempo: endSyncPoint?.syncPointValue?.modifiedTempo ?? synthBpm, - isStartMarker: false, - isEndMarker: true + markerType: SyncPointMarkerType.EndMarker, + ratioPosition: 1, + synthTick: synthTickPosition }); return markers; @@ -440,9 +532,9 @@ type MarkerDragInfo = { }; function computeMarkerInlineStyle( - m: MasterBarMarker, + m: SyncPointMarker, zoom: number, - draggingMarker: MasterBarMarker | null, + draggingMarker: SyncPointMarker | null, draggingMarkerInfo: MarkerDragInfo | null ): React.CSSProperties { let left = timePositionToX(m.syncTime, zoom); @@ -465,26 +557,26 @@ function updateSyncPointsAfterModification( ) { // find previous and next sync point (or start/end of the song) let startIndexForUpdate = Math.max(0, modifiedIndex - 1); - while (startIndexForUpdate > 0 && !s.masterBarMarkers[startIndexForUpdate].modifiedTempo) { + while (startIndexForUpdate > 0 && !s.syncPointMarkers[startIndexForUpdate].modifiedTempo) { startIndexForUpdate--; } - let nextIndexForUpdate = Math.min(s.masterBarMarkers.length - 1, modifiedIndex + 1); + let nextIndexForUpdate = Math.min(s.syncPointMarkers.length - 1, modifiedIndex + 1); while ( - nextIndexForUpdate < s.masterBarMarkers.length - 1 && - !s.masterBarMarkers[nextIndexForUpdate].modifiedTempo + nextIndexForUpdate < s.syncPointMarkers.length - 1 && + !s.syncPointMarkers[nextIndexForUpdate].modifiedTempo ) { nextIndexForUpdate++; } - const modifiedMarker = s.masterBarMarkers[modifiedIndex]; + const modifiedMarker = s.syncPointMarkers[modifiedIndex]; // update from previous to current if (startIndexForUpdate < modifiedIndex) { const previousMarker = cloneMarkers - ? { ...s.masterBarMarkers[startIndexForUpdate] } - : s.masterBarMarkers[startIndexForUpdate]; - s.masterBarMarkers[startIndexForUpdate] = previousMarker; + ? { ...s.syncPointMarkers[startIndexForUpdate] } + : s.syncPointMarkers[startIndexForUpdate]; + s.syncPointMarkers[startIndexForUpdate] = previousMarker; const synthDuration = modifiedMarker.synthTime - previousMarker.synthTime; const syncedDuration = modifiedMarker.syncTime - previousMarker.syncTime; const newBpmBefore = (synthDuration / syncedDuration) * previousMarker.synthBpm; @@ -492,29 +584,37 @@ function updateSyncPointsAfterModification( let syncedTimePosition = previousMarker.syncTime; for (let i = startIndexForUpdate; i < modifiedIndex; i++) { - const marker = cloneMarkers ? { ...s.masterBarMarkers[i] } : s.masterBarMarkers[i]; - s.masterBarMarkers[i] = marker; + const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; + s.syncPointMarkers[i] = marker; marker.syncTime = syncedTimePosition; - syncedTimePosition += ticksToMilliseconds(marker.synthTickDuration, newBpmBefore); + + if (i < modifiedIndex - 1) { + const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmBefore); + } } } if (!isDelete) { - const nextMarker = s.masterBarMarkers[nextIndexForUpdate]; + const nextMarker = s.syncPointMarkers[nextIndexForUpdate]; const synthDuration = nextMarker.synthTime - modifiedMarker.synthTime; const syncedDuration = nextMarker.syncTime - modifiedMarker.syncTime; const newBpmAfter = (synthDuration / syncedDuration) * modifiedMarker.synthBpm; modifiedMarker.modifiedTempo = newBpmAfter; - let syncedTimePosition = - modifiedMarker.syncTime + ticksToMilliseconds(modifiedMarker.synthTickDuration, newBpmAfter); + const tickDuration = s.syncPointMarkers[modifiedIndex + 1].synthTick - modifiedMarker.synthTick; + let syncedTimePosition = modifiedMarker.syncTime + ticksToMilliseconds(tickDuration, newBpmAfter); + for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) { - const marker = cloneMarkers ? { ...s.masterBarMarkers[i] } : s.masterBarMarkers[i]; - s.masterBarMarkers[i] = marker; + const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; + s.syncPointMarkers[i] = marker; marker.syncTime = syncedTimePosition; - syncedTimePosition += ticksToMilliseconds(marker.synthTickDuration, newBpmAfter); + if (i < nextIndexForUpdate - 1) { + const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmAfter); + } } } } @@ -539,10 +639,10 @@ export const MediaSyncEditor: React.FC = ({ api, score }) sampleRate: 44100, leftSamples: new Float32Array(0), rightSamples: new Float32Array(0), - masterBarMarkers: [] + syncPointMarkers: [] }); - const [draggingMarker, setDraggingMarker] = useState(null); + const [draggingMarker, setDraggingMarker] = useState(null); const [draggingMarkerInfo, setDraggingMarkerInfo] = useState(null); const [undoStack, setUndoStack] = useState({ undo: [], redo: [] }); const [shouldStoreToUndo, setStoreToUndo] = useState(false); @@ -565,7 +665,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) setAudioElement( (api.player!.output as alphaTab.synth.IAudioElementBackingTrackSynthOutput)?.audioElement ?? null ); - buildSyncPointInfo(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); + buildSyncPointInfoFromApi(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); setCreateInitialSyncPoints(false); }, [shouldCreateInitialSyncPoints] @@ -632,7 +732,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) } const syncPointLookup = new Map(); - for (const m of syncPointInfo.masterBarMarkers) { + for (const m of syncPointInfo.syncPointMarkers) { if (m.modifiedTempo) { let syncPoints = syncPointLookup.get(m.masterBarIndex); if (!syncPoints) { @@ -641,7 +741,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) } const automation = new alphaTab.model.Automation(); - automation.ratioPosition = m.isEndMarker ? 1 : 0; + automation.ratioPosition = m.ratioPosition; automation.type = alphaTab.model.AutomationType.SyncPoint; automation.syncPointValue = new alphaTab.model.SyncPointData(); automation.syncPointValue.modifiedTempo = m.modifiedTempo; @@ -693,7 +793,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }); }; - const toggleMarker = (marker: MasterBarMarker, e: React.MouseEvent) => { + const toggleMarker = (marker: SyncPointMarker, e: React.MouseEvent) => { e.stopPropagation(); e.preventDefault(); setStoreToUndo(true); @@ -704,18 +804,28 @@ export const MediaSyncEditor: React.FC = ({ api, score }) } // no removal of start and end marker - if (marker.isStartMarker || marker.isEndMarker) { + if ( + marker.markerType === SyncPointMarkerType.StartMarker || + marker.markerType === SyncPointMarkerType.EndMarker + ) { return s; } - const markerIndex = s!.masterBarMarkers.indexOf(marker); + const markerIndex = s!.syncPointMarkers.indexOf(marker); if (markerIndex === -1) { return s; } - const newS = { ...s, masterBarMarkers: [...s.masterBarMarkers] }; + const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; if (marker.modifiedTempo) { - newS.masterBarMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; + switch (marker.markerType) { + case SyncPointMarkerType.MasterBar: + newS.syncPointMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; + break; + case SyncPointMarkerType.Intermediate: + newS.syncPointMarkers.splice(markerIndex, 1); + break; + } updateSyncPointsAfterModification(markerIndex, newS, true, true); } else { updateSyncPointsAfterModification(markerIndex, newS, false, true); @@ -732,7 +842,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) e.stopPropagation(); const deltaX = draggingMarkerInfo!.endX - draggingMarkerInfo!.startX; - if (deltaX > dragThreshold || draggingMarker.modifiedTempo !== undefined) { + if (deltaX > dragThreshold || (draggingMarker.modifiedTempo !== undefined && Math.abs(deltaX) > 0)) { setStoreToUndo(true); setApplySyncPoints(true); setSyncPointInfo(s => { @@ -740,27 +850,28 @@ export const MediaSyncEditor: React.FC = ({ api, score }) return s; } - const markerIndex = s.masterBarMarkers.findIndex(m => m === draggingMarker); + const markerIndex = s.syncPointMarkers.findIndex(m => m === draggingMarker); const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; const deltaTime = deltaX / zoomedPixelPerMillisecond; const newTimePosition = draggingMarker.syncTime + deltaTime; - const newS = { ...s, masterBarMarkers: [...s.masterBarMarkers] }; + const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; // move the marker to the new position - newS.masterBarMarkers[markerIndex] = { - ...newS.masterBarMarkers[markerIndex], + newS.syncPointMarkers[markerIndex] = { + ...newS.syncPointMarkers[markerIndex], syncTime: Math.max(0, newTimePosition) }; updateSyncPointsAfterModification(markerIndex, newS, false, true); return newS; }); - setDraggingMarker(null); - setDraggingMarkerInfo(null); } + + setDraggingMarker(null); + setDraggingMarkerInfo(null); } }, [draggingMarker, draggingMarkerInfo] @@ -776,31 +887,60 @@ export const MediaSyncEditor: React.FC = ({ api, score }) return s; } - const index = syncPointInfo.masterBarMarkers.indexOf(draggingMarker!); + const index = syncPointInfo.syncPointMarkers.indexOf(draggingMarker!); if (index === -1) { return s; } let pageX = e.pageX; - if (index < syncPointInfo.masterBarMarkers.length - 1) { - const deltaX = pageX - s.startX; - const thisX = timePositionToX(draggingMarker!.syncTime, zoom); - const newX = thisX + deltaX; - - let nextMarkerIndex = index + 1; - while ( - nextMarkerIndex < syncPointInfo.masterBarMarkers.length - 1 && - !syncPointInfo.masterBarMarkers[nextMarkerIndex].modifiedTempo - ) { - nextMarkerIndex++; + const deltaX = pageX - s.startX; + + if (deltaX < 0) { + if (index > 0) { + const thisX = timePositionToX(draggingMarker!.syncTime, zoom); + const newX = thisX + deltaX; + + let previousMarkerIndex = index - 1; + if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { + while ( + previousMarkerIndex > 0 && + !syncPointInfo.syncPointMarkers[previousMarkerIndex].modifiedTempo + ) { + previousMarkerIndex--; + } + } + + const previousMarker = syncPointInfo.syncPointMarkers[previousMarkerIndex]; + const previousX = timePositionToX(previousMarker.syncTime, zoom); + const minX = previousX + dragLimit; + + if (newX < minX) { + pageX = s.startX - (thisX - minX); + } } - const nextMarker = syncPointInfo.masterBarMarkers[nextMarkerIndex]; - const nextX = timePositionToX(nextMarker.syncTime, zoom); - const maxX = nextX - dragLimit; + } else { + if (index < syncPointInfo.syncPointMarkers.length - 1) { + const thisX = timePositionToX(draggingMarker!.syncTime, zoom); + const newX = thisX + deltaX; + + let nextMarkerIndex = index + 1; + if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { + while ( + nextMarkerIndex < syncPointInfo.syncPointMarkers.length - 1 && + !syncPointInfo.syncPointMarkers[nextMarkerIndex].modifiedTempo + ) { + nextMarkerIndex++; + } + } - if (newX > maxX) { - pageX = s.startX + (maxX - thisX); + const nextMarker = syncPointInfo.syncPointMarkers[nextMarkerIndex]; + const nextX = timePositionToX(nextMarker.syncTime, zoom); + const maxX = nextX - dragLimit; + + if (newX > maxX) { + pageX = s.startX + (maxX - thisX); + } } } @@ -822,7 +962,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }; }, [draggingMarker, mouseUpListener, mouseMoveListener]); - const startMarkerDrag = (marker: MasterBarMarker, e: React.MouseEvent) => { + const startMarkerDrag = (marker: SyncPointMarker, e: React.MouseEvent) => { if (e.button !== 0 || marker.modifiedTempo === undefined) { return; } @@ -838,7 +978,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) redo: [] }); setStoreToUndo(true); - buildSyncPointInfo(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); + buildSyncPointInfoFromApi(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); setCreateInitialSyncPoints(false); }, [api]); @@ -1013,18 +1153,20 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }); }; - function buildMarkerLabel(m: MasterBarMarker): React.ReactNode { - if (m.isStartMarker) { - return 'Start'; - } - if (m.isEndMarker) { - return 'End'; - } - - if (m.occurence > 0) { - return `${m.masterBarIndex + 1} (${m.occurence + 1})`; + function buildMarkerLabel(m: SyncPointMarker): React.ReactNode { + switch (m.markerType) { + case SyncPointMarkerType.StartMarker: + return 'Start'; + case SyncPointMarkerType.EndMarker: + return 'End'; + case SyncPointMarkerType.MasterBar: + if (m.occurence > 0) { + return `${m.masterBarIndex + 1} (${m.occurence + 1})`; + } + return `${m.masterBarIndex + 1}`; + case SyncPointMarkerType.Intermediate: + return ''; } - return `${m.masterBarIndex + 1}`; } return ( @@ -1139,10 +1281,10 @@ export const MediaSyncEditor: React.FC = ({ api, score })
- {syncPointInfo.masterBarMarkers.map(m => ( + {syncPointInfo.syncPointMarkers.map(m => (
toggleMarker(m, e)} onMouseDown={e => { @@ -1151,7 +1293,7 @@ export const MediaSyncEditor: React.FC = ({ api, score })
{buildMarkerLabel(m)}
- {!m.isEndMarker && m.modifiedTempo && ( + {m.markerType !== SyncPointMarkerType.EndMarker && m.modifiedTempo && (
{m.modifiedTempo.toFixed(1)} bpm
)}
diff --git a/src/components/AlphaTabPlayground/styles.module.scss b/src/components/AlphaTabPlayground/styles.module.scss index 571bb6063..c48d4fb8a 100644 --- a/src/components/AlphaTabPlayground/styles.module.scss +++ b/src/components/AlphaTabPlayground/styles.module.scss @@ -439,13 +439,15 @@ transition: transform linear 0.1s; white-space: nowrap; padding-top: 50px; - font-size: 12px;; + font-size: 12px; + ; } & .sync-area-marker-wrap { position: absolute; top: 0; + &>.masterbar-marker { position: absolute; width: 20px; @@ -465,6 +467,7 @@ opacity: 1; } + &>.marker-label { height: 20px; white-space: nowrap; @@ -500,13 +503,33 @@ } } - &>.marker-line { flex-grow: 1; - width: 1px; - background: #444950; + width: 0; + border-right: 1px solid #444950; + } + + &.masterbar-marker-startmarker, + &.masterbar-marker-endmarker { + &>.marker-line { + border-right-style: solid; + border-right-width: 2px; + } } + &.masterbar-marker-masterbar { + &>.marker-line { + border-right-style: solid; + } + } + + &.masterbar-marker-intermediate { + &>.marker-line { + border-right-style: dashed; + } + } + + cursor: col-resize; &.has-sync-point { @@ -520,8 +543,6 @@ background: #4972a1; } } - - } } From 2be953c01d2b2d96f7be8ec4c3f0df89b5d26578 Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Thu, 22 May 2025 21:47:40 +0200 Subject: [PATCH 3/6] fix: wrong duration calculation and unique keys --- src/components/AlphaTabPlayground/media-sync-editor.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/AlphaTabPlayground/media-sync-editor.tsx b/src/components/AlphaTabPlayground/media-sync-editor.tsx index 149965b6a..9c75fe528 100644 --- a/src/components/AlphaTabPlayground/media-sync-editor.tsx +++ b/src/components/AlphaTabPlayground/media-sync-editor.tsx @@ -131,8 +131,8 @@ async function buildSyncPointInfoFromApi( marker.markerType = SyncPointMarkerType.StartMarker; } else if (changes.tick > masterBar.start) { marker.markerType = SyncPointMarkerType.Intermediate; - const duration = masterBar.start - masterBar.end; - marker.ratioPosition = changes.tick / duration; + const duration = masterBar.end - masterBar.start; + marker.ratioPosition = (changes.tick - masterBar.start) / duration; } if (changes.tempo !== synthBpm || marker.markerType === SyncPointMarkerType.StartMarker) { @@ -1116,7 +1116,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) const onLoadAudioFile = () => { const input = document.createElement('input'); input.type = 'file'; - input.accept = '.mp3,.ogg,*.wav,*.flac,*.aac'; + input.accept = '.mp3,.ogg,*.wav,*.flac,*.aac,*.mp4,*.mkv,*.avi,*.webm'; input.onchange = () => { if (input.files?.length === 1) { const reader = new FileReader(); @@ -1283,7 +1283,7 @@ export const MediaSyncEditor: React.FC = ({ api, score }) style={{ width: `${virtualWidth}px`, height: `${canvasSize[1]}px` }}> {syncPointInfo.syncPointMarkers.map(m => (
toggleMarker(m, e)} From 54616ed56a3ff4f132eef57d5443c8d3492953b3 Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Thu, 22 May 2025 23:55:10 +0200 Subject: [PATCH 4/6] refactor: Splitup and cleanup of code --- src/components/AlphaTabPlayground/helpers.ts | 63 + .../AlphaTabPlayground/media-sync-editor.tsx | 1237 ++--------------- .../AlphaTabPlayground/styles.module.scss | 197 +-- .../AlphaTabPlayground/sync-point-info.ts | 640 +++++++++ .../sync-point-marker-panel.tsx | 232 ++++ .../AlphaTabPlayground/waveform-canvas.tsx | 196 +++ 6 files changed, 1381 insertions(+), 1184 deletions(-) create mode 100644 src/components/AlphaTabPlayground/helpers.ts create mode 100644 src/components/AlphaTabPlayground/sync-point-info.ts create mode 100644 src/components/AlphaTabPlayground/sync-point-marker-panel.tsx create mode 100644 src/components/AlphaTabPlayground/waveform-canvas.tsx diff --git a/src/components/AlphaTabPlayground/helpers.ts b/src/components/AlphaTabPlayground/helpers.ts new file mode 100644 index 000000000..b81a80592 --- /dev/null +++ b/src/components/AlphaTabPlayground/helpers.ts @@ -0,0 +1,63 @@ +import { useState } from "react"; +import type { SyncPointInfo } from "./sync-point-info"; + +export function timePositionToX(pixelPerMilliseconds: number, + timePosition: number, zoom: number, leftPadding: number): number { + const zoomedPixelPerMilliseconds = pixelPerMilliseconds * zoom; + return timePosition * zoomedPixelPerMilliseconds + leftPadding; +} + +type UndoStack = { + undo: SyncPointInfo[]; + redo: SyncPointInfo[]; +}; + +export const useSyncPointInfoUndo = () => { + const [undoStack, setUndoStack] = useState({ undo: [], redo: [] }); + + return { + undo(callback: (info: SyncPointInfo) => void) { + setUndoStack(s => { + if (s.undo.length > 1) { + const newStack = { ...s }; + const undoState = newStack.undo.pop()!; + newStack.redo.push(undoState); + callback(newStack.undo.at(-1)!); + return newStack; + } + return s; + }); + }, + storeUndo(info: SyncPointInfo) { + setUndoStack(s => ({ + undo: [...s.undo, info], + redo: [] + })); + }, + redo(callback: (info: SyncPointInfo) => void) { + setUndoStack(s => { + if (s.redo.length > 0) { + const newStack = { ...s }; + const redoState = newStack.redo.pop()!; + newStack.undo.push(redoState); + callback(redoState); + return newStack; + } + return s; + }); + }, + get canUndo() { + // we always want to keep the initial state + return undoStack.undo.length > 1; + }, + get canRedo() { + return undoStack.redo.length > 0; + }, + resetUndo() { + setUndoStack({ + undo: [], + redo: [] + }); + } + } +}; \ No newline at end of file diff --git a/src/components/AlphaTabPlayground/media-sync-editor.tsx b/src/components/AlphaTabPlayground/media-sync-editor.tsx index 9c75fe528..57e7e271f 100644 --- a/src/components/AlphaTabPlayground/media-sync-editor.tsx +++ b/src/components/AlphaTabPlayground/media-sync-editor.tsx @@ -6,650 +6,33 @@ import styles from './styles.module.scss'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import * as solid from '@fortawesome/free-solid-svg-icons'; import * as brands from '@fortawesome/free-brands-svg-icons'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import useResizeObserver from '@react-hook/resize-observer'; import { useAlphaTabEvent } from '@site/src/hooks'; - -// TODO: cleanup the code (splitup helpers and components properly) +import { + applySyncPoints, + buildSyncPointInfoFromApi, + autoSync, + resetSyncPoints, + type SyncPointInfo +} from './sync-point-info'; +import { WaveformCanvas } from './waveform-canvas'; +import { SyncPointMarkerPanel } from './sync-point-marker-panel'; +import { timePositionToX, useSyncPointInfoUndo } from './helpers'; + +// General Settings for the UI +const pixelPerMilliseconds = 100 / 1000; +const leftPadding = 15; +const scrollThresholdPercent = 0.2; export type MediaSyncEditorProps = { api: alphaTab.AlphaTabApi; score: alphaTab.model.Score; }; -enum SyncPointMarkerType { - StartMarker = 0, - EndMarker = 1, - MasterBar = 2, - Intermediate = 3 -} - -type SyncPointMarker = { - syncTime: number; - - synthTime: number; - synthBpm: number; - synthTick: number; - - masterBarIndex: number; - ratioPosition: number; - occurence: number; - modifiedTempo?: number; - - markerType: SyncPointMarkerType; -}; - -type SyncPointInfo = { - endTick: number; - endTime: number; - sampleRate: number; - leftSamples: Float32Array; - rightSamples: Float32Array; - syncPointMarkers: SyncPointMarker[]; -}; - -function ticksToMilliseconds(tick: number, bpm: number): number { - return (tick * 60000.0) / (bpm * 960); -} - -async function buildSyncPointInfoFromApi( - api: alphaTab.AlphaTabApi, - createInitialSyncPoints: boolean -): Promise { - const tickCache = api.tickCache; - if (!tickCache || !api.score?.backingTrack?.rawAudioFile) { - return { - endTick: 0, - endTime: 0, - sampleRate: 0, - leftSamples: new Float32Array(0), - rightSamples: new Float32Array(0), - syncPointMarkers: [] - }; - } - - const audioContext = new AudioContext(); - const buffer = await audioContext.decodeAudioData(api.score!.backingTrack!.rawAudioFile.buffer.slice(0)); - const rawSamples: Float32Array[] = - buffer.numberOfChannels === 1 - ? [buffer.getChannelData(0), buffer.getChannelData(0)] - : [buffer.getChannelData(0), buffer.getChannelData(1)]; - - const sampleRate = audioContext.sampleRate; - const endTime = (rawSamples[0].length / sampleRate) * 1000; - - await audioContext.close(); - - const state: SyncPointInfo = { - endTick: api.tickCache.masterBars.at(-1)!.end, - syncPointMarkers: [], - sampleRate, - leftSamples: rawSamples[0], - rightSamples: rawSamples[1], - endTime - }; - - if (createInitialSyncPoints) { - // create initial sync points for all tempo changes to ensure the song and the - // backing track roughly align - let synthBpm = api.tickCache!.masterBars[0].tempoChanges[0].tempo; - let synthTimePosition = 0; - let synthTickPosition = 0; - - const syncPoints: SyncPointMarker[] = []; - - // first create all changes not respecting the song start and end - const occurences = new Map(); - for (const masterBar of api.tickCache!.masterBars) { - const occurence = occurences.get(masterBar.masterBar.index) ?? 0; - occurences.set(masterBar.masterBar.index, occurence + 1); - - // we are guaranteed to have a tempo change per master bar indicating its own tempo - // (even though its not a change) - for (const changes of masterBar.tempoChanges) { - const absoluteTick = changes.tick; - const tickOffset = absoluteTick - synthTickPosition; - if (tickOffset > 0) { - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition = absoluteTick; - synthTimePosition += timeOffset; - } - - const marker: SyncPointMarker = { - markerType: SyncPointMarkerType.MasterBar, - masterBarIndex: masterBar.masterBar.index, - occurence, - syncTime: synthTimePosition, - synthBpm, - synthTime: synthTimePosition, - modifiedTempo: undefined, - ratioPosition: 0, - synthTick: synthTickPosition - }; - - if (masterBar.start === 0) { - marker.markerType = SyncPointMarkerType.StartMarker; - } else if (changes.tick > masterBar.start) { - marker.markerType = SyncPointMarkerType.Intermediate; - const duration = masterBar.end - masterBar.start; - marker.ratioPosition = (changes.tick - masterBar.start) / duration; - } - - if (changes.tempo !== synthBpm || marker.markerType === SyncPointMarkerType.StartMarker) { - syncPoints.push(marker); - marker.modifiedTempo = changes.tempo; - } - - synthBpm = changes.tempo; - - state.syncPointMarkers.push(marker); - } - - const tickOffset = masterBar.end - synthTickPosition; - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition += tickOffset; - synthTimePosition += timeOffset; - } - - // end marker - const lastMasterBar = api.tickCache!.masterBars.at(-1)!; - state.syncPointMarkers.push({ - masterBarIndex: lastMasterBar.masterBar.index, - occurence: occurences.get(lastMasterBar.masterBar.index)!, - syncTime: synthTimePosition, - synthTime: synthTimePosition, - synthBpm, - modifiedTempo: synthBpm, - markerType: SyncPointMarkerType.EndMarker, - ratioPosition: 1, - synthTick: synthTickPosition - }); - - // with the final durations known, we can "squeeze" together the song - // from start and end (keeping the relative positions) - // and the other bars will be adjusted accordingly - const [songStart, songEnd] = findAudioStartAndEnd(state); - - const synthDuration = synthTimePosition; - const realDuration = songEnd - songStart; - const scaleFactor = realDuration / synthDuration; - - // 1st Pass: shift all tempo change markers relatively and calculate BPM - let syncTime = songStart; - for (let i = 0; i < syncPoints.length; i++) { - const syncPoint = syncPoints[i]; - - syncPoint.syncTime = syncTime; - - if (i < 0) { - const previousMarker = syncPoints[i - 1]; - const synthDuration = syncPoint.synthTime - previousMarker.synthTime; - const syncedDuration = syncPoint.syncTime - previousMarker.syncTime; - const newBpm = (synthDuration / syncedDuration) * previousMarker.synthBpm; - previousMarker.modifiedTempo = newBpm; - } - - const ownStart = syncPoint.synthTime; - const nextStart = i < syncPoints.length - 1 ? syncPoints[i + 1].synthTime : ownStart; - - const oldDuration = nextStart - ownStart; - const newDuration = oldDuration * scaleFactor; - - syncTime += newDuration; - } - - // 2nd Pass: adjust all in-between markers according to the new position - syncTime = songStart; - let syncedBpm = syncPoints[0].modifiedTempo!; - for (let i = 0; i < state.syncPointMarkers.length; i++) { - const marker = state.syncPointMarkers[i]; - marker.syncTime = syncTime; - - if (marker.modifiedTempo) { - syncedBpm = marker.modifiedTempo; - } - - if (i < state.syncPointMarkers.length - 1) { - const tickDiff = state.syncPointMarkers[i + 1].synthTick - marker.synthTick; - syncTime += ticksToMilliseconds(tickDiff, syncedBpm); - } - } - } else { - state.syncPointMarkers = buildSyncPointMarkers(api); - } - - return state; -} - -function resetSyncPoints(api: alphaTab.AlphaTabApi, state: SyncPointInfo): SyncPointInfo { - for (const b of api.score!.masterBars) { - b.syncPoints = undefined; - } - - return { - ...state, - syncPointMarkers: buildSyncPointMarkers(api) - }; -} - -function findAudioStartAndEnd(state: SyncPointInfo): [number, number] { - // once we have 1s non-silent audio we consider it as start (or inverted for end) - const nonSilentSamplesThreshold = 1 * state.sampleRate; - // we accept 200ms of silence inbetween audible samples - const silentSamplesThreshold = 0.2 * state.sampleRate; - // there can always be a bit of a noise. we require some amplitude - // proper would be to consider the Frequency and calculate the - const nonSilentAmplitudeThreshold = 0.001; - - // we limit the search to 10s (from start/end), proper audio should not exceed this - const searchThreshold = Math.min(10 * state.sampleRate, state.leftSamples.length * 0.1); - - let songStart = searchThreshold; - let songEnd = state.leftSamples.length - searchThreshold; - - // find start offset - let sampleIndex = 0; - - let nonSilentSamplesInSection = 0; - let silentSamplesInSequence = 0; - let sectionStart = 0; - - while (sampleIndex < songStart) { - if ( - Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || - Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold - ) { - // the first audible sample marks the potential start - if (nonSilentSamplesInSection === 0) { - sectionStart = sampleIndex; - } - nonSilentSamplesInSection++; - silentSamplesInSequence = 0; - } else { - silentSamplesInSequence++; - } - - // found more than X-samples silent, no start until here - if (silentSamplesInSequence > silentSamplesThreshold) { - // reset and start searching agian - sectionStart = sampleIndex + 1; - nonSilentSamplesInSection = 0; - silentSamplesInSequence = 0; - } - // found enough samples since section start which are audible, should be good - else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { - songStart = sectionStart; - break; - } - - sampleIndex++; - } - - // and same from the back - sampleIndex = state.leftSamples.length - 1; - nonSilentSamplesInSection = 0; - silentSamplesInSequence = 0; - sectionStart = sampleIndex; - - while (sampleIndex >= songEnd) { - if ( - Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || - Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold - ) { - if (nonSilentSamplesInSection === 0) { - sectionStart = sampleIndex; - } - nonSilentSamplesInSection++; - silentSamplesInSequence = 0; - } else { - silentSamplesInSequence++; - } - - if (silentSamplesInSequence > silentSamplesThreshold) { - sectionStart = sampleIndex - 1; - nonSilentSamplesInSection = 0; - silentSamplesInSequence = 0; - } else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { - songEnd = sectionStart; - break; - } - - sampleIndex--; - } - - return [(songStart / state.sampleRate) * 1000, (songEnd / state.sampleRate) * 1000]; -} - -function cropToAudio(state: SyncPointInfo): SyncPointInfo { - const [songStart, songEnd] = findAudioStartAndEnd(state); - const newState: SyncPointInfo = { - ...state, - syncPointMarkers: [...state.syncPointMarkers] - }; - // move first marker - newState.syncPointMarkers[0] = { - ...newState.syncPointMarkers[0], - syncTime: songStart - }; - updateSyncPointsAfterModification(0, newState, false, true); - - // move last marker - newState.syncPointMarkers[newState.syncPointMarkers.length - 1] = { - ...newState.syncPointMarkers[newState.syncPointMarkers.length - 1], - syncTime: songEnd - }; - updateSyncPointsAfterModification(newState.syncPointMarkers.length - 1, newState, false, true); - - return newState; -} - -function buildSyncPointMarkers(api: alphaTab.AlphaTabApi): SyncPointMarker[] { - const markers: SyncPointMarker[] = []; - - const occurences = new Map(); - let syncBpm = api.score!.tempo; - let syncLastTick = 0; - let syncLastMillisecondOffset = 0; - - let synthBpm = api.score!.tempo; - let synthTimePosition = 0; - let synthTickPosition = 0; - - for (const masterBar of api.tickCache!.masterBars) { - const occurence = occurences.get(masterBar.masterBar.index) ?? 0; - occurences.set(masterBar.masterBar.index, occurence + 1); - - const duration = masterBar.end - masterBar.start; - - if (masterBar.masterBar.syncPoints) { - // if we have sync points we have to correctly walk through the points and tempo changes - // and place the markers accordingly - - // TODO: create placeholder markers matching the time signature and relative offsets. - - let tempoChangeIndex = 0; - for (const syncPoint of masterBar.masterBar.syncPoints) { - if (syncPoint.syncPointValue!.barOccurence !== occurence) { - continue; - } - - const syncPointTick = masterBar.start + syncPoint.ratioPosition * duration; - - // first process all tempo change until this sync point - while ( - tempoChangeIndex < masterBar.tempoChanges.length && - masterBar.tempoChanges[tempoChangeIndex].tick <= syncPointTick - ) { - const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; - const absoluteTick = tempoChange.tick; - const tickOffset = absoluteTick - synthTickPosition; - if (tickOffset > 0) { - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition = absoluteTick; - synthTimePosition += timeOffset; - } - - synthBpm = tempoChange.tempo; - tempoChangeIndex++; - } - - // process time until sync point - const tickOffset = syncPointTick - synthTickPosition; - if (tickOffset > 0) { - synthTickPosition = syncPointTick; - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTimePosition += timeOffset; - } - - // create sync point marker - const newMarker: SyncPointMarker = { - masterBarIndex: masterBar.masterBar.index, - occurence: occurence, - syncTime: syncPoint.syncPointValue!.millisecondOffset, - synthTime: synthTimePosition, - synthBpm: masterBar.tempoChanges[0].tempo, - modifiedTempo: syncPoint!.syncPointValue!.modifiedTempo, - markerType: - syncPoint.ratioPosition === 0 - ? SyncPointMarkerType.MasterBar - : SyncPointMarkerType.Intermediate, - ratioPosition: syncPoint.ratioPosition, - synthTick: synthTickPosition - }; - if (syncPointTick === 0) { - newMarker.markerType = SyncPointMarkerType.StartMarker; - } - markers.push(newMarker); - - // remember values for artificially generated markers - syncBpm = syncPoint.syncPointValue!.modifiedTempo; - syncLastMillisecondOffset = syncPoint.syncPointValue!.millisecondOffset; - syncLastTick = masterBar.start; - } - - // process remaining tempo changes after all sync points - while (tempoChangeIndex < masterBar.tempoChanges.length) { - const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; - const absoluteTick = tempoChange.tick; - const tickOffset = absoluteTick - synthTickPosition; - if (tickOffset > 0) { - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition = absoluteTick; - synthTimePosition += timeOffset; - } - - synthBpm = tempoChange.tempo; - tempoChangeIndex++; - } - } else { - // TODO: Create intermediate markers matching time signature - - // if there are no sync points, we create a main masterbar sync point marker at start - let tickOffset = masterBar.start - syncLastTick; - - const newMarker: SyncPointMarker = { - masterBarIndex: masterBar.masterBar.index, - occurence: occurence, - syncTime: syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm), - synthTime: synthTimePosition, - synthTick: synthTickPosition, - synthBpm: masterBar.tempoChanges[0].tempo, - modifiedTempo: undefined, - markerType: masterBar.start === 0 ? SyncPointMarkerType.StartMarker : SyncPointMarkerType.MasterBar, - ratioPosition: 0 - }; - markers.push(newMarker); - - // and then we walk through the tempo changes - for (const changes of masterBar.tempoChanges) { - const absoluteTick = changes.tick; - const tickOffset = absoluteTick - synthTickPosition; - if (tickOffset > 0) { - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - - synthTickPosition = absoluteTick; - synthTimePosition += timeOffset; - } - - synthBpm = changes.tempo; - } - - // don't forget the part after the last tempo change - tickOffset = masterBar.end - synthTickPosition; - const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); - synthTickPosition += tickOffset; - synthTimePosition += timeOffset; - } - } - - // at the very end we create the end marker - const lastMasterBar = api.tickCache!.masterBars.at(-1)!; - const endSyncPoint = lastMasterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 1); - - const tickOffset = lastMasterBar.end - syncLastTick; - const endSyncPointTime = endSyncPoint - ? endSyncPoint.syncPointValue!.millisecondOffset - : syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm); - - markers.push({ - masterBarIndex: lastMasterBar.masterBar.index, - occurence: occurences.get(lastMasterBar.masterBar.index)!, - syncTime: endSyncPointTime, - synthTime: synthTimePosition, - synthBpm, - modifiedTempo: endSyncPoint?.syncPointValue?.modifiedTempo ?? synthBpm, - markerType: SyncPointMarkerType.EndMarker, - ratioPosition: 1, - synthTick: synthTickPosition - }); - - return markers; -} - -const pixelPerMilliseconds = 100 / 1000 /* 100px per 1000ms */; -const leftPadding = 15; -const barNumberHeight = 20; -const arrowHeight = 20; -const timeAxisHeight = 20; -const timeAxiSubSecondTickHeight = 5; -const barWidth = 1; -const timeAxisLineColor = '#A5A5A5'; -const waveFormColor = '#436d9d99'; -const font = '12px "Noto Sans"'; -const dragLimit = 10; -const dragThreshold = 5; -const scrollThresholdPercent = 0.2; - -function timePositionToX(timePosition: number, zoom: number): number { - const zoomedPixelPerMilliseconds = pixelPerMilliseconds * zoom; - return timePosition * zoomedPixelPerMilliseconds + leftPadding; -} - -type MarkerDragInfo = { - startX: number; - startY: number; - endX: number; -}; - -function computeMarkerInlineStyle( - m: SyncPointMarker, - zoom: number, - draggingMarker: SyncPointMarker | null, - draggingMarkerInfo: MarkerDragInfo | null -): React.CSSProperties { - let left = timePositionToX(m.syncTime, zoom); - - if (m === draggingMarker && draggingMarkerInfo) { - const deltaX = draggingMarkerInfo.endX - draggingMarkerInfo.startX; - left += deltaX; - } - - return { - left: `${left}px` - }; -} - -function updateSyncPointsAfterModification( - modifiedIndex: number, - s: SyncPointInfo, - isDelete: boolean, - cloneMarkers: boolean -) { - // find previous and next sync point (or start/end of the song) - let startIndexForUpdate = Math.max(0, modifiedIndex - 1); - while (startIndexForUpdate > 0 && !s.syncPointMarkers[startIndexForUpdate].modifiedTempo) { - startIndexForUpdate--; - } - - let nextIndexForUpdate = Math.min(s.syncPointMarkers.length - 1, modifiedIndex + 1); - while ( - nextIndexForUpdate < s.syncPointMarkers.length - 1 && - !s.syncPointMarkers[nextIndexForUpdate].modifiedTempo - ) { - nextIndexForUpdate++; - } - - const modifiedMarker = s.syncPointMarkers[modifiedIndex]; - - // update from previous to current - if (startIndexForUpdate < modifiedIndex) { - const previousMarker = cloneMarkers - ? { ...s.syncPointMarkers[startIndexForUpdate] } - : s.syncPointMarkers[startIndexForUpdate]; - s.syncPointMarkers[startIndexForUpdate] = previousMarker; - const synthDuration = modifiedMarker.synthTime - previousMarker.synthTime; - const syncedDuration = modifiedMarker.syncTime - previousMarker.syncTime; - const newBpmBefore = (synthDuration / syncedDuration) * previousMarker.synthBpm; - previousMarker.modifiedTempo = newBpmBefore; - - let syncedTimePosition = previousMarker.syncTime; - for (let i = startIndexForUpdate; i < modifiedIndex; i++) { - const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; - s.syncPointMarkers[i] = marker; - - marker.syncTime = syncedTimePosition; - - if (i < modifiedIndex - 1) { - const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; - syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmBefore); - } - } - } - - if (!isDelete) { - const nextMarker = s.syncPointMarkers[nextIndexForUpdate]; - const synthDuration = nextMarker.synthTime - modifiedMarker.synthTime; - const syncedDuration = nextMarker.syncTime - modifiedMarker.syncTime; - const newBpmAfter = (synthDuration / syncedDuration) * modifiedMarker.synthBpm; - modifiedMarker.modifiedTempo = newBpmAfter; - - const tickDuration = s.syncPointMarkers[modifiedIndex + 1].synthTick - modifiedMarker.synthTick; - let syncedTimePosition = modifiedMarker.syncTime + ticksToMilliseconds(tickDuration, newBpmAfter); - - for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) { - const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; - s.syncPointMarkers[i] = marker; - marker.syncTime = syncedTimePosition; - - if (i < nextIndexForUpdate - 1) { - const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; - syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmAfter); - } - } - } -} - -type UndoStack = { - undo: SyncPointInfo[]; - redo: SyncPointInfo[]; -}; - -export const MediaSyncEditor: React.FC = ({ api, score }) => { - const markerCanvas = useRef(null); - const waveFormCanvas = useRef(null); - const syncArea = useRef(null); - - const [canvasSize, setCanvasSize] = useState([0, 0]); - const [virtualWidth, setVirtualWidth] = useState(0); - const [zoom, setZoom] = useState(1); - - const [syncPointInfo, setSyncPointInfo] = useState({ - endTick: 0, - endTime: 0, - sampleRate: 44100, - leftSamples: new Float32Array(0), - rightSamples: new Float32Array(0), - syncPointMarkers: [] - }); - - const [draggingMarker, setDraggingMarker] = useState(null); - const [draggingMarkerInfo, setDraggingMarkerInfo] = useState(null); - const [undoStack, setUndoStack] = useState({ undo: [], redo: [] }); - const [shouldStoreToUndo, setStoreToUndo] = useState(false); - const [shouldApplySyncPoints, setApplySyncPoints] = useState(false); - const [shouldCreateInitialSyncPoints, setCreateInitialSyncPoints] = useState(false); - +const useAudioElementPlaybackTime = (api: alphaTab.AlphaTabApi) => { const [audioElement, setAudioElement] = useState(null); + const [playbackTime, setPlaybackTime] = useState(0); useEffect(() => { @@ -658,45 +41,22 @@ export const MediaSyncEditor: React.FC = ({ api, score }) ); }, [api.player!.output]); - useAlphaTabEvent( - api, - 'midiLoad', - () => { - setAudioElement( - (api.player!.output as alphaTab.synth.IAudioElementBackingTrackSynthOutput)?.audioElement ?? null - ); - buildSyncPointInfoFromApi(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); - setCreateInitialSyncPoints(false); - }, - [shouldCreateInitialSyncPoints] - ); - - useEffect(() => { - if (syncArea.current) { - const xPos = timePositionToX(playbackTime, zoom); - const canvasWidth = canvasSize[0]; - const threshold = canvasWidth * scrollThresholdPercent; - const scrollOffset = syncArea.current.scrollLeft; - - // is out of screen? - if (xPos < scrollOffset + threshold || xPos - scrollOffset > canvasWidth - threshold) { - syncArea.current.scrollTo({ - left: xPos - canvasWidth / 2, - behavior: 'smooth' - }); - } - } - }, [api, playbackTime, canvasSize, syncArea]); + useAlphaTabEvent(api, 'midiLoad', () => { + setAudioElement( + (api.player!.output as alphaTab.synth.IAudioElementBackingTrackSynthOutput)?.audioElement ?? null + ); + }); useEffect(() => { const updateWaveFormCursor = () => { - setPlaybackTime(audioElement!.currentTime * 1000); + if (audioElement) { + setPlaybackTime(audioElement.currentTime * 1000); + } }; let timeUpdate: number = 0; if (audioElement) { - console.log('Audio element', audioElement); audioElement.addEventListener('timeupdate', updateWaveFormCursor); audioElement.addEventListener('durationchange', updateWaveFormCursor); audioElement.addEventListener('seeked', updateWaveFormCursor); @@ -710,7 +70,6 @@ export const MediaSyncEditor: React.FC = ({ api, score }) return () => { if (audioElement) { - console.log('unregister Audio element', audioElement); audioElement.removeEventListener('timeupdate', updateWaveFormCursor); audioElement.removeEventListener('durationchange', updateWaveFormCursor); audioElement.removeEventListener('seeked', updateWaveFormCursor); @@ -719,399 +78,97 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }; }, [audioElement]); - useEffect(() => { - if (!syncPointInfo) { - return; - } - if (shouldStoreToUndo) { - setUndoStack(s => ({ - undo: [...s.undo, syncPointInfo], - redo: [] - })); - setStoreToUndo(false); - } - - const syncPointLookup = new Map(); - for (const m of syncPointInfo.syncPointMarkers) { - if (m.modifiedTempo) { - let syncPoints = syncPointLookup.get(m.masterBarIndex); - if (!syncPoints) { - syncPoints = []; - syncPointLookup.set(m.masterBarIndex, syncPoints); - } - - const automation = new alphaTab.model.Automation(); - automation.ratioPosition = m.ratioPosition; - automation.type = alphaTab.model.AutomationType.SyncPoint; - automation.syncPointValue = new alphaTab.model.SyncPointData(); - automation.syncPointValue.modifiedTempo = m.modifiedTempo; - automation.syncPointValue.millisecondOffset = m.syncTime; - automation.syncPointValue.barOccurence = m.occurence; - syncPoints.push(automation); - } - } - - if (shouldApplySyncPoints) { - console.log('Apply Sync points', syncPointLookup); - - // remember and set again the tick position after sync point update - // this will ensure the cursor and player seek accordingly with keeping the cursor - // where it is currently shown on the notation. - const tickPosition = api.tickPosition; - for (const masterBar of score.masterBars) { - masterBar.syncPoints = syncPointLookup.get(masterBar.index); - } - api.updateSyncPoints(); - api.tickPosition = tickPosition; - setApplySyncPoints(false); - } - }, [syncPointInfo]); + return playbackTime; +}; - const undo = () => { - setUndoStack(s => { - const newStack = { ...s }; - if (newStack.undo.length > 0) { - const undoState = newStack.undo.pop()!; - newStack.redo.push(undoState); - setApplySyncPoints(true); - setSyncPointInfo(newStack.undo.at(-1)!); - } - return newStack; - }); - }; +export const MediaSyncEditor: React.FC = ({ api, score }) => { + const syncArea = useRef(null); - const redo = () => { - setUndoStack(s => { - const newStack = { ...s }; - if (newStack.redo.length > 0) { - const redoState = newStack.redo.pop()!; - newStack.undo.push(redoState); - setApplySyncPoints(true); - setSyncPointInfo(redoState); - } - return newStack; - }); - }; + const [canvasSize, setCanvasSize] = useState([0, 0]); + const [virtualWidth, setVirtualWidth] = useState(0); + const [zoom, setZoom] = useState(1); + const [scrolOffset, setScrollOffset] = useState(0); - const toggleMarker = (marker: SyncPointMarker, e: React.MouseEvent) => { - e.stopPropagation(); - e.preventDefault(); - setStoreToUndo(true); - setApplySyncPoints(true); - setSyncPointInfo(s => { - if (!s) { - return s; - } + const [syncPointInfo, setSyncPointInfo] = useState({ + endTick: 0, + endTime: 0, + sampleRate: 44100, + leftSamples: new Float32Array(0), + rightSamples: new Float32Array(0), + syncPointMarkers: [] + }); - // no removal of start and end marker - if ( - marker.markerType === SyncPointMarkerType.StartMarker || - marker.markerType === SyncPointMarkerType.EndMarker - ) { - return s; - } + const shouldStoreToUndo = useRef(false); + const shouldApplySyncPoints = useRef(false); + const shouldCreateInitialSyncPoints = useRef(false); - const markerIndex = s!.syncPointMarkers.indexOf(marker); - if (markerIndex === -1) { - return s; - } + const undo = useSyncPointInfoUndo(); + const playbackTime = useAudioElementPlaybackTime(api); - const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; - if (marker.modifiedTempo) { - switch (marker.markerType) { - case SyncPointMarkerType.MasterBar: - newS.syncPointMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; - break; - case SyncPointMarkerType.Intermediate: - newS.syncPointMarkers.splice(markerIndex, 1); - break; - } - updateSyncPointsAfterModification(markerIndex, newS, true, true); - } else { - updateSyncPointsAfterModification(markerIndex, newS, false, true); - } + // Sync Point Info building and update - return newS; - }); + const initFromApi = () => { + undo.resetUndo(); + shouldStoreToUndo.current = true; + buildSyncPointInfoFromApi(api, shouldCreateInitialSyncPoints.current).then(x => setSyncPointInfo(x)); + shouldCreateInitialSyncPoints.current = false; }; - const mouseUpListener = useCallback( - (e: MouseEvent) => { - if (draggingMarker) { - e.preventDefault(); - e.stopPropagation(); - - const deltaX = draggingMarkerInfo!.endX - draggingMarkerInfo!.startX; - if (deltaX > dragThreshold || (draggingMarker.modifiedTempo !== undefined && Math.abs(deltaX) > 0)) { - setStoreToUndo(true); - setApplySyncPoints(true); - setSyncPointInfo(s => { - if (!s) { - return s; - } - - const markerIndex = s.syncPointMarkers.findIndex(m => m === draggingMarker); - - const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; - const deltaTime = deltaX / zoomedPixelPerMillisecond; - - const newTimePosition = draggingMarker.syncTime + deltaTime; - - const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; - - // move the marker to the new position - newS.syncPointMarkers[markerIndex] = { - ...newS.syncPointMarkers[markerIndex], - syncTime: Math.max(0, newTimePosition) - }; - - updateSyncPointsAfterModification(markerIndex, newS, false, true); - return newS; - }); - } - - setDraggingMarker(null); - setDraggingMarkerInfo(null); - } - }, - [draggingMarker, draggingMarkerInfo] - ); - - const mouseMoveListener = useCallback( - (e: MouseEvent) => { - e.preventDefault(); - e.stopPropagation(); - - setDraggingMarkerInfo(s => { - if (!s || !syncPointInfo) { - return s; - } - - const index = syncPointInfo.syncPointMarkers.indexOf(draggingMarker!); - if (index === -1) { - return s; - } - - let pageX = e.pageX; - const deltaX = pageX - s.startX; - - if (deltaX < 0) { - if (index > 0) { - const thisX = timePositionToX(draggingMarker!.syncTime, zoom); - const newX = thisX + deltaX; - - let previousMarkerIndex = index - 1; - if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { - while ( - previousMarkerIndex > 0 && - !syncPointInfo.syncPointMarkers[previousMarkerIndex].modifiedTempo - ) { - previousMarkerIndex--; - } - } - - const previousMarker = syncPointInfo.syncPointMarkers[previousMarkerIndex]; - const previousX = timePositionToX(previousMarker.syncTime, zoom); - const minX = previousX + dragLimit; - - if (newX < minX) { - pageX = s.startX - (thisX - minX); - } - } - - } else { - if (index < syncPointInfo.syncPointMarkers.length - 1) { - const thisX = timePositionToX(draggingMarker!.syncTime, zoom); - const newX = thisX + deltaX; - - let nextMarkerIndex = index + 1; - if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { - while ( - nextMarkerIndex < syncPointInfo.syncPointMarkers.length - 1 && - !syncPointInfo.syncPointMarkers[nextMarkerIndex].modifiedTempo - ) { - nextMarkerIndex++; - } - } - - const nextMarker = syncPointInfo.syncPointMarkers[nextMarkerIndex]; - const nextX = timePositionToX(nextMarker.syncTime, zoom); - const maxX = nextX - dragLimit; - - if (newX > maxX) { - pageX = s.startX + (maxX - thisX); - } - } - } - - return { ...s, endX: pageX }; - }); + useAlphaTabEvent( + api, + 'midiLoad', + () => { + initFromApi(); }, - [draggingMarker, syncPointInfo] + [shouldCreateInitialSyncPoints] ); useEffect(() => { - if (draggingMarker) { - document.addEventListener('mouseup', mouseUpListener); - document.addEventListener('mousemove', mouseMoveListener); - } - - return () => { - document.removeEventListener('mouseup', mouseUpListener); - document.removeEventListener('mousemove', mouseMoveListener); - }; - }, [draggingMarker, mouseUpListener, mouseMoveListener]); - - const startMarkerDrag = (marker: SyncPointMarker, e: React.MouseEvent) => { - if (e.button !== 0 || marker.modifiedTempo === undefined) { - return; - } - e.preventDefault(); - e.stopPropagation(); - setDraggingMarkerInfo(() => ({ startX: e.pageX, startY: e.pageY, endX: e.pageX })); - setDraggingMarker(() => marker); - }; - - useEffect(() => { - setUndoStack({ - undo: [], - redo: [] - }); - setStoreToUndo(true); - buildSyncPointInfoFromApi(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); - setCreateInitialSyncPoints(false); + initFromApi(); }, [api]); - const drawWaveform = () => { - const can = waveFormCanvas.current; - if (!syncPointInfo || !can) { - return; + useEffect(() => { + // store to undo if needed + if (shouldStoreToUndo.current) { + undo.storeUndo(syncPointInfo); + shouldStoreToUndo.current = false; } - const ctx = can.getContext('2d')!; - ctx.clearRect(0, 0, can.width, can.height); - ctx.save(); - - const waveFormY = barNumberHeight + arrowHeight; - const halfHeight = ((can.height - waveFormY - timeAxisHeight) / 2) | 0; - - // frame - ctx.fillStyle = timeAxisLineColor; - ctx.fillRect(0, waveFormY + 2 * halfHeight, can.width, 1); - ctx.fillRect(0, barNumberHeight, can.width, 1); - ctx.fillRect(0, waveFormY, can.width, 1); - ctx.fillRect(0, waveFormY + halfHeight, can.width, 1); - - // waveform - ctx.translate(-syncArea.current!.scrollLeft, 0); - - ctx.beginPath(); - - const startX = Math.max(syncArea.current!.scrollLeft - leftPadding, 0); - const endX = startX + can.width + leftPadding; - - const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; - const samplesPerPixel = syncPointInfo.sampleRate / (zoomedPixelPerMillisecond * 1000); - - for (let x = startX; x < endX; x += barWidth) { - const startSample = (x * samplesPerPixel) | 0; - const endSample = ((x + barWidth) * samplesPerPixel) | 0; - - let maxTop = 0; - let maxBottom = 0; - for (let sample = startSample; sample <= endSample; sample++) { - const magnitudeTop = Math.abs(syncPointInfo.leftSamples[sample] || 0); - const magnitudeBottom = Math.abs(syncPointInfo.rightSamples[sample] || 0); - if (magnitudeTop > maxTop) { - maxTop = magnitudeTop; - } - if (magnitudeBottom > maxBottom) { - maxBottom = magnitudeBottom; - } - } - - const topBarHeight = Math.min(halfHeight, Math.round(maxTop * halfHeight)); - const bottomBarHeight = Math.min(halfHeight, Math.round(maxBottom * halfHeight)); - const barHeight = topBarHeight + bottomBarHeight || 1; - ctx.rect(x + leftPadding, waveFormY + (halfHeight - topBarHeight), barWidth, barHeight); + // apply if needed + if (shouldApplySyncPoints) { + applySyncPoints(api, syncPointInfo); + shouldApplySyncPoints.current = false; } + }, [syncPointInfo]); - ctx.fillStyle = waveFormColor; - ctx.fill(); - - // time axis - ctx.save(); - - ctx.fillStyle = timeAxisLineColor; - ctx.font = font; - ctx.textAlign = 'left'; - ctx.textBaseline = 'bottom'; - - const timeAxisY = waveFormY + 2 * halfHeight; - const leftTimeSecond = Math.floor((startX - leftPadding) / zoomedPixelPerMillisecond / 1000); - const rightTimeSecond = Math.ceil(endX / zoomedPixelPerMillisecond / 1000); - - const leftTime = leftTimeSecond * 1000; - const rightTime = rightTimeSecond * 1000; - - let time = leftTime; - while (time <= rightTime) { - const timeX = timePositionToX(time, zoom); - ctx.fillRect(timeX, timeAxisY, 1, timeAxisHeight); - - const totalSeconds = Math.abs(time / 1000); - - const minutes = Math.floor(totalSeconds / 60); - const seconds = Math.floor(totalSeconds - minutes * 60); - - const sign = time < 0 ? '-' : ''; - const minutesText = minutes.toString().padStart(2, '0'); - const secondsText = seconds.toString().padStart(2, '0'); - - ctx.fillText(`${sign}${minutesText}:${secondsText}`, timeX + 3, timeAxisY + timeAxisHeight); - - const nextSecond = time + 1000; - while (time < nextSecond) { - const subSecondX = timePositionToX(time, zoom); - ctx.fillRect(subSecondX, timeAxisY, 1, timeAxiSubSecondTickHeight); + // cursor handling + useEffect(() => { + if (syncArea.current) { + const xPos = timePositionToX(pixelPerMilliseconds, playbackTime, zoom, leftPadding); + const canvasWidth = canvasSize[0]; + const threshold = canvasWidth * scrollThresholdPercent; + const scrollOffset = syncArea.current.scrollLeft; - time += 100; + // is out of screen? + if (xPos < scrollOffset + threshold || xPos - scrollOffset > canvasWidth - threshold) { + syncArea.current.scrollTo({ + left: xPos - canvasWidth / 2, + behavior: 'smooth' + }); } } + }, [playbackTime, canvasSize, syncArea]); - ctx.restore(); - ctx.restore(); - }; - - useEffect(() => { - if (waveFormCanvas.current) { - waveFormCanvas.current.width = canvasSize[0]; - waveFormCanvas.current.height = canvasSize[1]; - } - drawWaveform(); - }, [waveFormCanvas, canvasSize]); - + // UI parts useResizeObserver(syncArea, entry => { - setCanvasSize(s => [entry.contentRect.width, entry.contentRect.height]); + setCanvasSize([entry.contentRect.width, entry.contentRect.height]); }); useEffect(() => { if (syncPointInfo) { setVirtualWidth(s => pixelPerMilliseconds * syncPointInfo.endTime * zoom); } - drawWaveform(); - }, [markerCanvas, syncPointInfo, zoom]); - - useEffect(() => { - if (shouldCreateInitialSyncPoints) { - // clear any potential sync points - for (const m of score.masterBars) { - m.syncPoints = undefined; - } - api.updateSettings(); - api.loadMidiForScore(); - } - }, [shouldCreateInitialSyncPoints]); + }, [syncPointInfo, zoom]); const onLoadAudioFile = () => { const input = document.createElement('input'); @@ -1126,8 +183,15 @@ export const MediaSyncEditor: React.FC = ({ api, score }) score.backingTrack.rawAudioFile = new Uint8Array(e.target!.result as ArrayBuffer); // create a fresh set of sync points upon load (start->end) - setApplySyncPoints(true); - setCreateInitialSyncPoints(true); + shouldApplySyncPoints.current = true; + shouldCreateInitialSyncPoints.current = true; + + // clear any potential sync points + for (const m of score.masterBars) { + m.syncPoints = undefined; + } + api.updateSettings(); + api.loadMidiForScore(); // will fire the initialization above once ready. }; reader.readAsArrayBuffer(input.files[0]); } @@ -1138,37 +202,21 @@ export const MediaSyncEditor: React.FC = ({ api, score }) }; const onResetSyncPoints = () => { - setApplySyncPoints(true); - setStoreToUndo(true); + shouldApplySyncPoints.current = true; + shouldStoreToUndo.current = true; setSyncPointInfo(s => { return s ? resetSyncPoints(api, s) : s; }); }; - const onCropToAudio = () => { - setApplySyncPoints(true); - setStoreToUndo(true); + const onAutoSync = () => { + shouldApplySyncPoints.current = true; + shouldStoreToUndo.current = true; setSyncPointInfo(s => { - return s ? cropToAudio(s) : s; + return s ? autoSync(s, api) : s; }); }; - function buildMarkerLabel(m: SyncPointMarker): React.ReactNode { - switch (m.markerType) { - case SyncPointMarkerType.StartMarker: - return 'Start'; - case SyncPointMarkerType.EndMarker: - return 'End'; - case SyncPointMarkerType.MasterBar: - if (m.occurence > 0) { - return `${m.masterBarIndex + 1} (${m.occurence + 1})`; - } - return `${m.masterBarIndex + 1}`; - case SyncPointMarkerType.Intermediate: - return ''; - } - } - return (
@@ -1213,9 +261,9 @@ export const MediaSyncEditor: React.FC = ({ api, score }) onClick={e => { e.preventDefault(); e.stopPropagation(); - onCropToAudio(); + onAutoSync(); }}> - Crop to Headable Audio + Automatic Sync with Tempo Changes and Audio @@ -1255,9 +303,12 @@ export const MediaSyncEditor: React.FC = ({ api, score }) type="button" data-tooltip-id="tooltip-playground" data-tooltip-content="Undo" - disabled={undoStack.undo.length <= 1} + disabled={!undo.canUndo} onClick={() => { - undo(); + undo.undo(i => { + shouldApplySyncPoints.current = true; + setSyncPointInfo(i); + }); }}> @@ -1267,43 +318,57 @@ export const MediaSyncEditor: React.FC = ({ api, score }) type="button" data-tooltip-id="tooltip-playground" data-tooltip-content="Redo" - disabled={undoStack.redo.length === 0} + disabled={!undo.canRedo} onClick={() => { - redo(); + undo.redo(i => { + shouldApplySyncPoints.current = true; + setSyncPointInfo(i); + }); }}>
-
drawWaveform()}> +
setScrollOffset((e.target as HTMLDivElement).scrollLeft)}>
- -
-
- {syncPointInfo.syncPointMarkers.map(m => ( -
toggleMarker(m, e)} - onMouseDown={e => { - startMarkerDrag(m, e); - }}> -
{buildMarkerLabel(m)}
-
-
- {m.markerType !== SyncPointMarkerType.EndMarker && m.modifiedTempo && ( -
{m.modifiedTempo.toFixed(1)} bpm
- )} -
-
-
- ))} +
+ { + shouldApplySyncPoints.current = true; + shouldStoreToUndo.current = true; + setSyncPointInfo(newInfo); + }} + />
diff --git a/src/components/AlphaTabPlayground/styles.module.scss b/src/components/AlphaTabPlayground/styles.module.scss index c48d4fb8a..716e01e9a 100644 --- a/src/components/AlphaTabPlayground/styles.module.scss +++ b/src/components/AlphaTabPlayground/styles.module.scss @@ -425,6 +425,10 @@ position: sticky; top: 0; left: 0; + + &>canvas { + font: 12px "Noto Sans"; + } } & .sync-area-playback-cursor { @@ -440,111 +444,108 @@ white-space: nowrap; padding-top: 50px; font-size: 12px; - ; + } + } +} + +.sync-area-marker-wrap { + position: absolute; + top: 0; + + &>.masterbar-marker { + position: absolute; + width: 20px; + top: 0; + bottom: 20px; + transform: translateX(-50%); + user-select: none; + + display: flex; + flex-direction: column; + align-items: center; + font-size: 12px; + + opacity: 0.4; + + &:hover { + opacity: 1; + } + + + &>.marker-label { + height: 20px; + white-space: nowrap; + display: flex; + align-items: center; } - & .sync-area-marker-wrap { - position: absolute; - top: 0; + &>.marker-head { + height: 20px; + width: 20px; + display: grid; + align-items: center; + grid-template-columns: 1fr; + grid-template-rows: 1fr; + + &>* { + grid-area: 1 / 1 / 2 / 2; + } - &>.masterbar-marker { - position: absolute; + &>.marker-arrow { width: 20px; - top: 0; - bottom: 20px; - transform: translateX(-50%); - user-select: none; - - display: flex; - flex-direction: column; - align-items: center; - font-size: 12px; - - opacity: 0.4; - - &:hover { - opacity: 1; - } - - - &>.marker-label { - height: 20px; - white-space: nowrap; - display: flex; - align-items: center; - } - - - &>.marker-head { - height: 20px; - width: 20px; - display: grid; - align-items: center; - grid-template-columns: 1fr; - grid-template-rows: 1fr; - - &>* { - grid-area: 1 / 1 / 2 / 2; - } - - &>.marker-arrow { - width: 20px; - height: 20px; - clip-path: polygon(0 0, 100% 0, 50% 100%); - background: #444950; - } - - &>.marker-tempo { - white-space: nowrap; - text-align: left; - transform: translateX(25px); - pointer-events: none; - } - } - - &>.marker-line { - flex-grow: 1; - width: 0; - border-right: 1px solid #444950; - } - - &.masterbar-marker-startmarker, - &.masterbar-marker-endmarker { - &>.marker-line { - border-right-style: solid; - border-right-width: 2px; - } - } - - &.masterbar-marker-masterbar { - &>.marker-line { - border-right-style: solid; - } - } - - &.masterbar-marker-intermediate { - &>.marker-line { - border-right-style: dashed; - } - } - - - cursor: col-resize; - - &.has-sync-point { - opacity: 0.75; - - &:hover { - opacity: 1; - } - - & .marker-arrow { - background: #4972a1; - } - } + height: 20px; + clip-path: polygon(0 0, 100% 0, 50% 100%); + background: #444950; + } + + &>.marker-tempo { + white-space: nowrap; + text-align: left; + transform: translateX(25px); + pointer-events: none; + } + } + + &>.marker-line { + flex-grow: 1; + width: 0; + border-right: 1px solid #444950; + } + + &.masterbar-marker-startmarker, + &.masterbar-marker-endmarker { + &>.marker-line { + border-right-style: solid; + border-right-width: 2px; } } + &.masterbar-marker-masterbar { + &>.marker-line { + border-right-style: solid; + } + } + + &.masterbar-marker-intermediate { + &>.marker-line { + border-right-style: dashed; + } + } + + + cursor: col-resize; + + &.has-sync-point { + opacity: 0.75; + + &:hover { + opacity: 1; + } + + & .marker-arrow { + background: #4972a1; + } + } } } \ No newline at end of file diff --git a/src/components/AlphaTabPlayground/sync-point-info.ts b/src/components/AlphaTabPlayground/sync-point-info.ts new file mode 100644 index 000000000..4719f87c1 --- /dev/null +++ b/src/components/AlphaTabPlayground/sync-point-info.ts @@ -0,0 +1,640 @@ +import * as alphaTab from '@coderline/alphatab'; + +export enum SyncPointMarkerType { + StartMarker = 0, + EndMarker = 1, + MasterBar = 2, + Intermediate = 3 +} + +export type SyncPointMarker = { + syncTime: number; + + synthTime: number; + synthBpm: number; + synthTick: number; + + masterBarIndex: number; + ratioPosition: number; + occurence: number; + modifiedTempo?: number; + + markerType: SyncPointMarkerType; +}; + +export type SyncPointInfo = { + endTick: number; + endTime: number; + sampleRate: number; + leftSamples: Float32Array; + rightSamples: Float32Array; + syncPointMarkers: SyncPointMarker[]; +}; + + +function ticksToMilliseconds(tick: number, bpm: number): number { + return (tick * 60000.0) / (bpm * 960); +} + +export async function buildSyncPointInfoFromApi( + api: alphaTab.AlphaTabApi, + createInitialSyncPoints: boolean +): Promise { + const tickCache = api.tickCache; + if (!tickCache || !api.score?.backingTrack?.rawAudioFile) { + return { + endTick: 0, + endTime: 0, + sampleRate: 0, + leftSamples: new Float32Array(0), + rightSamples: new Float32Array(0), + syncPointMarkers: [] + }; + } + + const audioContext = new AudioContext(); + const buffer = await audioContext.decodeAudioData(api.score!.backingTrack!.rawAudioFile.buffer.slice(0)); + const rawSamples: Float32Array[] = + buffer.numberOfChannels === 1 + ? [buffer.getChannelData(0), buffer.getChannelData(0)] + : [buffer.getChannelData(0), buffer.getChannelData(1)]; + + const sampleRate = audioContext.sampleRate; + const endTime = (rawSamples[0].length / sampleRate) * 1000; + + await audioContext.close(); + + const state: SyncPointInfo = { + endTick: api.tickCache.masterBars.at(-1)!.end, + syncPointMarkers: [], + sampleRate, + leftSamples: rawSamples[0], + rightSamples: rawSamples[1], + endTime + }; + + if (createInitialSyncPoints) { + return autoSync(state, api); + } + + state.syncPointMarkers = buildSyncPointMarkers(api); + return state; +} + +function buildSyncPointMarkers(api: alphaTab.AlphaTabApi): SyncPointMarker[] { + const markers: SyncPointMarker[] = []; + + const occurences = new Map(); + let syncBpm = api.score!.tempo; + let syncLastTick = 0; + let syncLastMillisecondOffset = 0; + + let synthBpm = api.score!.tempo; + let synthTimePosition = 0; + let synthTickPosition = 0; + + for (const masterBar of api.tickCache!.masterBars) { + const occurence = occurences.get(masterBar.masterBar.index) ?? 0; + occurences.set(masterBar.masterBar.index, occurence + 1); + + const duration = masterBar.end - masterBar.start; + + if (masterBar.masterBar.syncPoints) { + // if we have sync points we have to correctly walk through the points and tempo changes + // and place the markers accordingly + + // TODO: create placeholder markers matching the time signature and relative offsets. + + let tempoChangeIndex = 0; + for (const syncPoint of masterBar.masterBar.syncPoints) { + if (syncPoint.syncPointValue!.barOccurence !== occurence) { + continue; + } + + const syncPointTick = masterBar.start + syncPoint.ratioPosition * duration; + + // first process all tempo change until this sync point + while ( + tempoChangeIndex < masterBar.tempoChanges.length && + masterBar.tempoChanges[tempoChangeIndex].tick <= syncPointTick + ) { + const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; + const absoluteTick = tempoChange.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + synthBpm = tempoChange.tempo; + tempoChangeIndex++; + } + + // process time until sync point + const tickOffset = syncPointTick - synthTickPosition; + if (tickOffset > 0) { + synthTickPosition = syncPointTick; + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTimePosition += timeOffset; + } + + // create sync point marker + const newMarker: SyncPointMarker = { + masterBarIndex: masterBar.masterBar.index, + occurence: occurence, + syncTime: syncPoint.syncPointValue!.millisecondOffset, + synthTime: synthTimePosition, + synthBpm: masterBar.tempoChanges[0].tempo, + modifiedTempo: syncPoint!.syncPointValue!.modifiedTempo, + markerType: + syncPoint.ratioPosition === 0 + ? SyncPointMarkerType.MasterBar + : SyncPointMarkerType.Intermediate, + ratioPosition: syncPoint.ratioPosition, + synthTick: synthTickPosition + }; + if (syncPointTick === 0) { + newMarker.markerType = SyncPointMarkerType.StartMarker; + } + markers.push(newMarker); + + // remember values for artificially generated markers + syncBpm = syncPoint.syncPointValue!.modifiedTempo; + syncLastMillisecondOffset = syncPoint.syncPointValue!.millisecondOffset; + syncLastTick = masterBar.start; + } + + // process remaining tempo changes after all sync points + while (tempoChangeIndex < masterBar.tempoChanges.length) { + const tempoChange = masterBar.tempoChanges[tempoChangeIndex]; + const absoluteTick = tempoChange.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + synthBpm = tempoChange.tempo; + tempoChangeIndex++; + } + } else { + // TODO: Create intermediate markers matching time signature + + // if there are no sync points, we create a main masterbar sync point marker at start + let tickOffset = masterBar.start - syncLastTick; + + const newMarker: SyncPointMarker = { + masterBarIndex: masterBar.masterBar.index, + occurence: occurence, + syncTime: syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm), + synthTime: synthTimePosition, + synthTick: synthTickPosition, + synthBpm: masterBar.tempoChanges[0].tempo, + modifiedTempo: undefined, + markerType: masterBar.start === 0 ? SyncPointMarkerType.StartMarker : SyncPointMarkerType.MasterBar, + ratioPosition: 0 + }; + + if (newMarker.markerType === SyncPointMarkerType.StartMarker) { + newMarker.modifiedTempo = syncBpm; + } + + markers.push(newMarker); + + // and then we walk through the tempo changes + for (const changes of masterBar.tempoChanges) { + const absoluteTick = changes.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + synthBpm = changes.tempo; + } + + // don't forget the part after the last tempo change + tickOffset = masterBar.end - synthTickPosition; + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition += tickOffset; + synthTimePosition += timeOffset; + } + } + + // at the very end we create the end marker + const lastMasterBar = api.tickCache!.masterBars.at(-1)!; + const endSyncPoint = lastMasterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 1); + + const tickOffset = lastMasterBar.end - syncLastTick; + const endSyncPointTime = endSyncPoint + ? endSyncPoint.syncPointValue!.millisecondOffset + : syncLastMillisecondOffset + ticksToMilliseconds(tickOffset, syncBpm); + + markers.push({ + masterBarIndex: lastMasterBar.masterBar.index, + occurence: occurences.get(lastMasterBar.masterBar.index)!, + syncTime: endSyncPointTime, + synthTime: synthTimePosition, + synthBpm, + modifiedTempo: endSyncPoint?.syncPointValue?.modifiedTempo ?? synthBpm, + markerType: SyncPointMarkerType.EndMarker, + ratioPosition: 1, + synthTick: synthTickPosition + }); + + return markers; +} + +function findAudioStartAndEnd(state: SyncPointInfo): [number, number] { + // once we have 1s non-silent audio we consider it as start (or inverted for end) + const nonSilentSamplesThreshold = 1 * state.sampleRate; + // we accept 200ms of silence inbetween audible samples + const silentSamplesThreshold = 0.2 * state.sampleRate; + // there can always be a bit of a noise. we require some amplitude + // proper would be to consider the Frequency and calculate the + const nonSilentAmplitudeThreshold = 0.001; + + // we limit the search to 10s (from start/end), proper audio should not exceed this + const searchThreshold = Math.min(10 * state.sampleRate, state.leftSamples.length * 0.1); + + let songStart = searchThreshold; + let songEnd = state.leftSamples.length - searchThreshold; + + // find start offset + let sampleIndex = 0; + + let nonSilentSamplesInSection = 0; + let silentSamplesInSequence = 0; + let sectionStart = 0; + + while (sampleIndex < songStart) { + if ( + Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || + Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold + ) { + // the first audible sample marks the potential start + if (nonSilentSamplesInSection === 0) { + sectionStart = sampleIndex; + } + nonSilentSamplesInSection++; + silentSamplesInSequence = 0; + } else { + silentSamplesInSequence++; + } + + // found more than X-samples silent, no start until here + if (silentSamplesInSequence > silentSamplesThreshold) { + // reset and start searching agian + sectionStart = sampleIndex + 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + } + // found enough samples since section start which are audible, should be good + else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { + songStart = sectionStart; + break; + } + + sampleIndex++; + } + + // and same from the back + sampleIndex = state.leftSamples.length - 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + sectionStart = sampleIndex; + + while (sampleIndex >= songEnd) { + if ( + Math.abs(state.leftSamples[sampleIndex]) >= nonSilentAmplitudeThreshold || + Math.abs(state.rightSamples[sampleIndex]) >= nonSilentAmplitudeThreshold + ) { + if (nonSilentSamplesInSection === 0) { + sectionStart = sampleIndex; + } + nonSilentSamplesInSection++; + silentSamplesInSequence = 0; + } else { + silentSamplesInSequence++; + } + + if (silentSamplesInSequence > silentSamplesThreshold) { + sectionStart = sampleIndex - 1; + nonSilentSamplesInSection = 0; + silentSamplesInSequence = 0; + } else if (nonSilentSamplesInSection > nonSilentSamplesThreshold) { + songEnd = sectionStart; + break; + } + + sampleIndex--; + } + + return [(songStart / state.sampleRate) * 1000, (songEnd / state.sampleRate) * 1000]; +} + +export function autoSync(oldState: SyncPointInfo, api: alphaTab.AlphaTabApi): SyncPointInfo { + const state: SyncPointInfo = { + endTick: api.tickCache!.masterBars.at(-1)!.end, + syncPointMarkers: [], + sampleRate: oldState.sampleRate, + leftSamples: oldState.leftSamples, + rightSamples: oldState.rightSamples, + endTime: oldState.endTime + }; + + // create initial sync points for all tempo changes to ensure the song and the + // backing track roughly align + let synthBpm = api.tickCache!.masterBars[0].tempoChanges[0].tempo; + let synthTimePosition = 0; + let synthTickPosition = 0; + + const syncPoints: SyncPointMarker[] = []; + + // first create all changes not respecting the song start and end + const occurences = new Map(); + for (const masterBar of api.tickCache!.masterBars) { + const occurence = occurences.get(masterBar.masterBar.index) ?? 0; + occurences.set(masterBar.masterBar.index, occurence + 1); + + // we are guaranteed to have a tempo change per master bar indicating its own tempo + // (even though its not a change) + for (const changes of masterBar.tempoChanges) { + const absoluteTick = changes.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + const marker: SyncPointMarker = { + markerType: SyncPointMarkerType.MasterBar, + masterBarIndex: masterBar.masterBar.index, + occurence, + syncTime: synthTimePosition, + synthBpm, + synthTime: synthTimePosition, + modifiedTempo: undefined, + ratioPosition: 0, + synthTick: synthTickPosition + }; + + if (masterBar.start === 0) { + marker.markerType = SyncPointMarkerType.StartMarker; + } else if (changes.tick > masterBar.start) { + marker.markerType = SyncPointMarkerType.Intermediate; + const duration = masterBar.end - masterBar.start; + marker.ratioPosition = (changes.tick - masterBar.start) / duration; + } + + if (changes.tempo !== synthBpm || marker.markerType === SyncPointMarkerType.StartMarker) { + syncPoints.push(marker); + marker.modifiedTempo = changes.tempo; + } + + synthBpm = changes.tempo; + + state.syncPointMarkers.push(marker); + } + + const tickOffset = masterBar.end - synthTickPosition; + const timeOffset = ticksToMilliseconds(tickOffset, synthBpm); + synthTickPosition += tickOffset; + synthTimePosition += timeOffset; + } + + // end marker + const lastMasterBar = api.tickCache!.masterBars.at(-1)!; + state.syncPointMarkers.push({ + masterBarIndex: lastMasterBar.masterBar.index, + occurence: occurences.get(lastMasterBar.masterBar.index)!, + syncTime: synthTimePosition, + synthTime: synthTimePosition, + synthBpm, + modifiedTempo: synthBpm, + markerType: SyncPointMarkerType.EndMarker, + ratioPosition: 1, + synthTick: synthTickPosition + }); + + // with the final durations known, we can "squeeze" together the song + // from start and end (keeping the relative positions) + // and the other bars will be adjusted accordingly + const [songStart, songEnd] = findAudioStartAndEnd(state); + + const synthDuration = synthTimePosition; + const realDuration = songEnd - songStart; + const scaleFactor = realDuration / synthDuration; + + // 1st Pass: shift all tempo change markers relatively and calculate BPM + let syncTime = songStart; + for (let i = 0; i < syncPoints.length; i++) { + const syncPoint = syncPoints[i]; + + syncPoint.syncTime = syncTime; + + if (i < 0) { + const previousMarker = syncPoints[i - 1]; + const synthDuration = syncPoint.synthTime - previousMarker.synthTime; + const syncedDuration = syncPoint.syncTime - previousMarker.syncTime; + const newBpm = (synthDuration / syncedDuration) * previousMarker.synthBpm; + previousMarker.modifiedTempo = newBpm; + } + + const ownStart = syncPoint.synthTime; + const nextStart = i < syncPoints.length - 1 ? syncPoints[i + 1].synthTime : ownStart; + + const oldDuration = nextStart - ownStart; + const newDuration = oldDuration * scaleFactor; + + syncTime += newDuration; + } + + // 2nd Pass: adjust all in-between markers according to the new position + syncTime = songStart; + let syncedBpm = syncPoints[0].modifiedTempo!; + for (let i = 0; i < state.syncPointMarkers.length; i++) { + const marker = state.syncPointMarkers[i]; + marker.syncTime = syncTime; + + if (marker.modifiedTempo) { + syncedBpm = marker.modifiedTempo; + } + + if (i < state.syncPointMarkers.length - 1) { + const tickDiff = state.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncTime += ticksToMilliseconds(tickDiff, syncedBpm); + } + } + + return state; +} + +export function updateSyncPointsAfterModification( + modifiedIndex: number, + s: SyncPointInfo, + isDelete: boolean, + cloneMarkers: boolean +) { + // find previous and next sync point (or start/end of the song) + let startIndexForUpdate = Math.max(0, modifiedIndex - 1); + while (startIndexForUpdate > 0 && !s.syncPointMarkers[startIndexForUpdate].modifiedTempo) { + startIndexForUpdate--; + } + + let nextIndexForUpdate = Math.min(s.syncPointMarkers.length - 1, modifiedIndex + 1); + while ( + nextIndexForUpdate < s.syncPointMarkers.length - 1 && + !s.syncPointMarkers[nextIndexForUpdate].modifiedTempo + ) { + nextIndexForUpdate++; + } + + const modifiedMarker = s.syncPointMarkers[modifiedIndex]; + + // update from previous to current + if (startIndexForUpdate < modifiedIndex) { + const previousMarker = cloneMarkers + ? { ...s.syncPointMarkers[startIndexForUpdate] } + : s.syncPointMarkers[startIndexForUpdate]; + s.syncPointMarkers[startIndexForUpdate] = previousMarker; + const synthDuration = modifiedMarker.synthTime - previousMarker.synthTime; + const syncedDuration = modifiedMarker.syncTime - previousMarker.syncTime; + const newBpmBefore = (synthDuration / syncedDuration) * previousMarker.synthBpm; + previousMarker.modifiedTempo = newBpmBefore; + + let syncedTimePosition = previousMarker.syncTime; + for (let i = startIndexForUpdate; i < modifiedIndex; i++) { + const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; + s.syncPointMarkers[i] = marker; + + marker.syncTime = syncedTimePosition; + + if (i < modifiedIndex - 1) { + const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmBefore); + } + } + } + + if (!isDelete && nextIndexForUpdate > modifiedIndex) { + const nextMarker = s.syncPointMarkers[nextIndexForUpdate]; + const synthDuration = nextMarker.synthTime - modifiedMarker.synthTime; + const syncedDuration = nextMarker.syncTime - modifiedMarker.syncTime; + const newBpmAfter = syncedDuration > 0 ? (synthDuration / syncedDuration) * modifiedMarker.synthBpm : 0; + modifiedMarker.modifiedTempo = newBpmAfter; + + const tickDuration = nextMarker.synthTick - modifiedMarker.synthTick; + let syncedTimePosition = modifiedMarker.syncTime + ticksToMilliseconds(tickDuration, newBpmAfter); + + for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) { + const marker = cloneMarkers ? { ...s.syncPointMarkers[i] } : s.syncPointMarkers[i]; + s.syncPointMarkers[i] = marker; + marker.syncTime = syncedTimePosition; + + if (i < nextIndexForUpdate - 1) { + const tickDuration = s.syncPointMarkers[i + 1].synthTick - marker.synthTick; + syncedTimePosition += ticksToMilliseconds(tickDuration, newBpmAfter); + } + } + } +} + +export function moveMarker(s: SyncPointInfo, marker: SyncPointMarker, newTimePosition: number): SyncPointInfo { + const markerIndex = s.syncPointMarkers.findIndex(m => m === marker); + + const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; + + // move the marker to the new position + newS.syncPointMarkers[markerIndex] = { + ...newS.syncPointMarkers[markerIndex], + syncTime: Math.max(0, newTimePosition) + }; + + updateSyncPointsAfterModification(markerIndex, newS, false, true); + return newS; +} + +export function toggleMarker(s: SyncPointInfo, marker: SyncPointMarker): SyncPointInfo { + // no removal of start and end marker + if ( + marker.markerType === SyncPointMarkerType.StartMarker || + marker.markerType === SyncPointMarkerType.EndMarker + ) { + return s; + } + + const markerIndex = s!.syncPointMarkers.indexOf(marker); + if (markerIndex === -1) { + return s; + } + + const newS: SyncPointInfo = { ...s, syncPointMarkers: [...s.syncPointMarkers] }; + if (marker.modifiedTempo) { + switch (marker.markerType) { + case SyncPointMarkerType.MasterBar: + newS.syncPointMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; + break; + case SyncPointMarkerType.Intermediate: + newS.syncPointMarkers.splice(markerIndex, 1); + break; + } + updateSyncPointsAfterModification(markerIndex, newS, true, true); + } else { + updateSyncPointsAfterModification(markerIndex, newS, false, true); + } + + return newS; +} + +export function resetSyncPoints(api: alphaTab.AlphaTabApi, state: SyncPointInfo): SyncPointInfo { + for (const b of api.score!.masterBars) { + b.syncPoints = undefined; + } + + return { + ...state, + syncPointMarkers: buildSyncPointMarkers(api) + }; +} + +export function applySyncPoints(api: alphaTab.AlphaTabApi, syncPointInfo: SyncPointInfo) { + const syncPointLookup = new Map(); + for (const m of syncPointInfo.syncPointMarkers) { + if (m.modifiedTempo) { + let syncPoints = syncPointLookup.get(m.masterBarIndex); + if (!syncPoints) { + syncPoints = []; + syncPointLookup.set(m.masterBarIndex, syncPoints); + } + + const automation = new alphaTab.model.Automation(); + automation.ratioPosition = m.ratioPosition; + automation.type = alphaTab.model.AutomationType.SyncPoint; + automation.syncPointValue = new alphaTab.model.SyncPointData(); + automation.syncPointValue.modifiedTempo = m.modifiedTempo; + automation.syncPointValue.millisecondOffset = m.syncTime; + automation.syncPointValue.barOccurence = m.occurence; + syncPoints.push(automation); + } + } + + // remember and set again the tick position after sync point update + // this will ensure the cursor and player seek accordingly with keeping the cursor + // where it is currently shown on the notation. + const tickPosition = api.tickPosition; + for (const masterBar of api.score!.masterBars) { + masterBar.syncPoints = syncPointLookup.get(masterBar.index); + } + api.updateSyncPoints(); + api.tickPosition = tickPosition; +} + + + + diff --git a/src/components/AlphaTabPlayground/sync-point-marker-panel.tsx b/src/components/AlphaTabPlayground/sync-point-marker-panel.tsx new file mode 100644 index 000000000..9e6cc4470 --- /dev/null +++ b/src/components/AlphaTabPlayground/sync-point-marker-panel.tsx @@ -0,0 +1,232 @@ +import type React from 'react'; +import { + moveMarker, + type SyncPointInfo, + type SyncPointMarker, + SyncPointMarkerType, + toggleMarker +} from './sync-point-info'; +import styles from './styles.module.scss'; +import { timePositionToX } from './helpers'; +import { useCallback, useEffect, useState } from 'react'; + +export type SyncPointMarkerPanelProps = { + syncPointInfo: SyncPointInfo; + onSyncPointInfoChanged(syncPointInfo: SyncPointInfo): void; + + zoom: number; + width: number; + height: number; + pixelPerMilliseconds: number; + leftPadding: number; +}; + +type MarkerDragInfo = { + startX: number; + startY: number; + endX: number; +}; + +const dragLimit = 10; +const dragThreshold = 5; + +const buildMarkerLabel = (m: SyncPointMarker): React.ReactNode => { + switch (m.markerType) { + case SyncPointMarkerType.StartMarker: + return 'Start'; + case SyncPointMarkerType.EndMarker: + return 'End'; + case SyncPointMarkerType.MasterBar: + if (m.occurence > 0) { + return `${m.masterBarIndex + 1} (${m.occurence + 1})`; + } + return `${m.masterBarIndex + 1}`; + case SyncPointMarkerType.Intermediate: + return ''; + } +}; + +const computeMarkerInlineStyle = ( + m: SyncPointMarker, + props: SyncPointMarkerPanelProps, + draggingMarker: SyncPointMarker | null, + draggingMarkerInfo: MarkerDragInfo | null +): React.CSSProperties => { + let left = timePositionToX(props.pixelPerMilliseconds, m.syncTime, props.zoom, props.leftPadding); + + if (m === draggingMarker && draggingMarkerInfo) { + const deltaX = draggingMarkerInfo.endX - draggingMarkerInfo.startX; + left += deltaX; + } + + return { + left: `${left}px` + }; +}; + +export const SyncPointMarkerPanel: React.FC = props => { + const { syncPointInfo, onSyncPointInfoChanged, width, height, zoom, pixelPerMilliseconds, leftPadding } = props; + + const [draggingMarker, setDraggingMarker] = useState(null); + const [draggingMarkerInfo, setDraggingMarkerInfo] = useState(null); + + const onToggleMarker = (marker: SyncPointMarker, e: React.MouseEvent) => { + e.stopPropagation(); + e.preventDefault(); + onSyncPointInfoChanged(toggleMarker(syncPointInfo, marker)); + }; + + const startMarkerDrag = (marker: SyncPointMarker, e: React.MouseEvent) => { + if (e.button !== 0 || marker.modifiedTempo === undefined) { + return; + } + e.preventDefault(); + e.stopPropagation(); + setDraggingMarkerInfo(() => ({ startX: e.pageX, startY: e.pageY, endX: e.pageX })); + setDraggingMarker(() => marker); + }; + + const mouseUpListener = useCallback( + (e: MouseEvent) => { + if (draggingMarker) { + e.preventDefault(); + e.stopPropagation(); + + const deltaX = draggingMarkerInfo!.endX - draggingMarkerInfo!.startX; + if (deltaX > dragThreshold || (draggingMarker.modifiedTempo !== undefined && Math.abs(deltaX) > 0)) { + const zoomedPixelPerMillisecond = pixelPerMilliseconds * zoom; + const deltaTime = deltaX / zoomedPixelPerMillisecond; + const newTimePosition = draggingMarker.syncTime + deltaTime; + onSyncPointInfoChanged(moveMarker(syncPointInfo, draggingMarker, newTimePosition)); + } + + setDraggingMarker(null); + setDraggingMarkerInfo(null); + } + }, + [draggingMarker, draggingMarkerInfo] + ); + + const mouseMoveListener = useCallback( + (e: MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + + setDraggingMarkerInfo(s => { + if (!s || !syncPointInfo) { + return s; + } + + const index = syncPointInfo.syncPointMarkers.indexOf(draggingMarker!); + if (index === -1) { + return s; + } + + let pageX = e.pageX; + const deltaX = pageX - s.startX; + + if (deltaX < 0) { + if (index > 0) { + const thisX = timePositionToX( + pixelPerMilliseconds, + draggingMarker!.syncTime, + zoom, + leftPadding + ); + const newX = thisX + deltaX; + + let previousMarkerIndex = index - 1; + if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { + while ( + previousMarkerIndex > 0 && + !syncPointInfo.syncPointMarkers[previousMarkerIndex].modifiedTempo + ) { + previousMarkerIndex--; + } + } + + const previousMarker = syncPointInfo.syncPointMarkers[previousMarkerIndex]; + const previousX = timePositionToX( + pixelPerMilliseconds, + previousMarker.syncTime, + zoom, + leftPadding + ); + const minX = previousX + dragLimit; + + if (newX < minX) { + pageX = s.startX - (thisX - minX); + } + } + } else { + if (index < syncPointInfo.syncPointMarkers.length - 1) { + const thisX = timePositionToX( + pixelPerMilliseconds, + draggingMarker!.syncTime, + zoom, + leftPadding + ); + const newX = thisX + deltaX; + + let nextMarkerIndex = index + 1; + if (draggingMarker!.markerType !== SyncPointMarkerType.Intermediate) { + while ( + nextMarkerIndex < syncPointInfo.syncPointMarkers.length - 1 && + !syncPointInfo.syncPointMarkers[nextMarkerIndex].modifiedTempo + ) { + nextMarkerIndex++; + } + } + + const nextMarker = syncPointInfo.syncPointMarkers[nextMarkerIndex]; + const nextX = timePositionToX(pixelPerMilliseconds, nextMarker.syncTime, zoom, leftPadding); + const maxX = nextX - dragLimit; + + if (newX > maxX) { + pageX = s.startX + (maxX - thisX); + } + } + } + + return { ...s, endX: pageX }; + }); + }, + [draggingMarker, syncPointInfo] + ); + + useEffect(() => { + if (draggingMarker) { + document.addEventListener('mouseup', mouseUpListener); + document.addEventListener('mousemove', mouseMoveListener); + } + + return () => { + document.removeEventListener('mouseup', mouseUpListener); + document.removeEventListener('mousemove', mouseMoveListener); + }; + }, [draggingMarker, mouseUpListener, mouseMoveListener]); + + return ( +
+ {syncPointInfo.syncPointMarkers.map(m => ( +
onToggleMarker(m, e)} + onMouseDown={e => { + startMarkerDrag(m, e); + }}> +
{buildMarkerLabel(m)}
+
+
+ {m.markerType !== SyncPointMarkerType.EndMarker && m.modifiedTempo && ( +
{m.modifiedTempo.toFixed(1)} bpm
+ )} +
+
+
+ ))} +
+ ); +}; diff --git a/src/components/AlphaTabPlayground/waveform-canvas.tsx b/src/components/AlphaTabPlayground/waveform-canvas.tsx new file mode 100644 index 000000000..e808f566d --- /dev/null +++ b/src/components/AlphaTabPlayground/waveform-canvas.tsx @@ -0,0 +1,196 @@ +import type React from 'react'; +import { useEffect, useRef } from 'react'; +import { timePositionToX } from './helpers'; + +export type WaveformCanvasProps = { + barNumberHeight?: number; + arrowHeight?: number; + + timeAxisHeight?: number; + timeAxisSubSecondTickHeight?: number; + timeAxisLineColor?: string; + + waveFormColor?: string; + pixelPerMilliseconds: number; + + leftPadding?: number; + zoom?: number; + + scrollOffset?: number; + + sampleRate: number; + leftSamples: Float32Array; + rightSamples: Float32Array; + + width: number; + height: number; +}; + +const defaultProps = { + barNumberHeight: 20, + arrowHeight: 20, + + timeAxisHeight: 20, + timeAxisSubSecondTickHeight: 5, + timeAxisLineColor: '#A5A5A5', + + waveFormColor: '#436d9d99', + + leftPadding: 0, + zoom: 1, + + scrollOffset: 0 +} satisfies Partial; + +type DrawWaveFormOptions = typeof defaultProps & WaveformCanvasProps; + +type CommonDrawInfo = { + waveFormY: number; + halfHeight: number; + startX: number; + endX: number; + zoomedPixelPerMillisecond: number; + samplesPerPixel: number; +}; + +const drawFrame = (ctx: CanvasRenderingContext2D, options: DrawWaveFormOptions, drawInfo: CommonDrawInfo) => { + ctx.fillStyle = options.timeAxisLineColor; + ctx.fillRect(0, drawInfo.waveFormY + 2 * drawInfo.halfHeight, ctx.canvas.width, 1); + ctx.fillRect(0, options.barNumberHeight, ctx.canvas.width, 1); + ctx.fillRect(0, drawInfo.waveFormY, ctx.canvas.width, 1); + ctx.fillRect(0, drawInfo.waveFormY + drawInfo.halfHeight, ctx.canvas.width, 1); +}; + +const drawSamples = (ctx: CanvasRenderingContext2D, options: DrawWaveFormOptions, drawInfo: CommonDrawInfo) => { + ctx.save(); + + ctx.translate(-options.scrollOffset, 0); + + ctx.beginPath(); + + const startX = Math.max(options.scrollOffset - options.leftPadding, 0); + const endX = startX + ctx.canvas.width + options.leftPadding; + + const zoomedPixelPerMillisecond = options.pixelPerMilliseconds * options.zoom; + const samplesPerPixel = options.sampleRate / (zoomedPixelPerMillisecond * 1000); + + for (let x = startX; x < endX; x++) { + const startSample = (x * samplesPerPixel) | 0; + const endSample = ((x + 1) * samplesPerPixel) | 0; + + let maxTop = 0; + let maxBottom = 0; + for (let sample = startSample; sample <= endSample; sample++) { + const magnitudeTop = Math.abs(options.leftSamples[sample] || 0); + const magnitudeBottom = Math.abs(options.rightSamples[sample] || 0); + if (magnitudeTop > maxTop) { + maxTop = magnitudeTop; + } + if (magnitudeBottom > maxBottom) { + maxBottom = magnitudeBottom; + } + } + + const topBarHeight = Math.min(drawInfo.halfHeight, Math.round(maxTop * drawInfo.halfHeight)); + const bottomBarHeight = Math.min(drawInfo.halfHeight, Math.round(maxBottom * drawInfo.halfHeight)); + const barHeight = topBarHeight + bottomBarHeight || 1; + ctx.rect(x + options.leftPadding, drawInfo.waveFormY + (drawInfo.halfHeight - topBarHeight), 1, barHeight); + } + + ctx.fillStyle = options.waveFormColor; + ctx.fill(); + + ctx.restore(); +}; + +const drawTimeAxis = (ctx: CanvasRenderingContext2D, options: DrawWaveFormOptions, drawInfo: CommonDrawInfo) => { + ctx.save(); + ctx.translate(-options.scrollOffset, 0); + + ctx.fillStyle = options.timeAxisLineColor; + const style = window.getComputedStyle(ctx.canvas); + ctx.font = style.font; + ctx.textAlign = 'left'; + ctx.textBaseline = 'bottom'; + + const timeAxisY = drawInfo.waveFormY + 2 * drawInfo.halfHeight; + const leftTimeSecond = Math.floor( + (drawInfo.startX - options.leftPadding) / drawInfo.zoomedPixelPerMillisecond / 1000 + ); + const rightTimeSecond = Math.ceil(drawInfo.endX / drawInfo.zoomedPixelPerMillisecond / 1000); + + const leftTime = leftTimeSecond * 1000; + const rightTime = rightTimeSecond * 1000; + + let time = leftTime; + while (time <= rightTime) { + const timeX = timePositionToX(options.pixelPerMilliseconds, time, options.zoom, options.leftPadding); + ctx.fillRect(timeX, timeAxisY, 1, options.timeAxisHeight); + + const totalSeconds = Math.abs(time / 1000); + + const minutes = Math.floor(totalSeconds / 60); + const seconds = Math.floor(totalSeconds - minutes * 60); + + const sign = time < 0 ? '-' : ''; + const minutesText = minutes.toString().padStart(2, '0'); + const secondsText = seconds.toString().padStart(2, '0'); + + ctx.fillText(`${sign}${minutesText}:${secondsText}`, timeX + 3, timeAxisY + options.timeAxisHeight); + + const nextSecond = time + 1000; + while (time < nextSecond) { + const subSecondX = timePositionToX(options.pixelPerMilliseconds, time, options.zoom, options.leftPadding); + + ctx.fillRect(subSecondX, timeAxisY, 1, options.timeAxisSubSecondTickHeight); + + time += 100; + } + } + + ctx.restore(); +}; +const drawWaveform = (can: HTMLCanvasElement, options: DrawWaveFormOptions) => { + const ctx = can.getContext('2d')!; + ctx.clearRect(0, 0, can.width, can.height); + ctx.save(); + + const waveFormY = options.barNumberHeight + options.arrowHeight; + const halfHeight = ((ctx.canvas.height - waveFormY - options.timeAxisHeight) / 2) | 0; + const startX = Math.max(options.scrollOffset - options.leftPadding, 0); + const endX = startX + ctx.canvas.width + options.leftPadding; + const zoomedPixelPerMillisecond = options.pixelPerMilliseconds * options.zoom; + const samplesPerPixel = options.sampleRate / (zoomedPixelPerMillisecond * 1000); + + const drawInfo: CommonDrawInfo = { + waveFormY, + halfHeight, + startX, + endX, + zoomedPixelPerMillisecond, + samplesPerPixel + }; + + drawFrame(ctx, options, drawInfo); + drawSamples(ctx, options, drawInfo); + drawTimeAxis(ctx, options, drawInfo); +}; + +export const WaveformCanvas: React.FC = props => { + const waveFormCanvas = useRef(null); + + const realProps = { + ...defaultProps, + ...props + }; + + useEffect(() => { + if (waveFormCanvas.current) { + waveFormCanvas.current.width = props.width; + waveFormCanvas.current.height = props.height; + drawWaveform(waveFormCanvas.current, realProps); + } + }, [props, waveFormCanvas]); + + return ; +}; From d5a49e04e38d50a40d4729457bb12569c2a569a6 Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Fri, 23 May 2025 00:17:25 +0200 Subject: [PATCH 5/6] perf: do not re-render waveform --- .../AlphaTabPlayground/waveform-canvas.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/components/AlphaTabPlayground/waveform-canvas.tsx b/src/components/AlphaTabPlayground/waveform-canvas.tsx index e808f566d..2d64c65be 100644 --- a/src/components/AlphaTabPlayground/waveform-canvas.tsx +++ b/src/components/AlphaTabPlayground/waveform-canvas.tsx @@ -62,6 +62,9 @@ const drawFrame = (ctx: CanvasRenderingContext2D, options: DrawWaveFormOptions, }; const drawSamples = (ctx: CanvasRenderingContext2D, options: DrawWaveFormOptions, drawInfo: CommonDrawInfo) => { + // NOTE: this is not very efficient. + // we likely should render main parts of the waveform once and reuse the drawn images? + // e.g. render multiple PNG chunks and simply show them. (re-draw on zoom) ctx.save(); ctx.translate(-options.scrollOffset, 0); @@ -184,13 +187,21 @@ export const WaveformCanvas: React.FC = props => { ...props }; + // watch all props by converting them to JSON, all props affect display + // but watching props directly causes re-draw on every component render. + const watchedProps = JSON.stringify({ + ...realProps, + leftSamples: null!, + rightSamples: null! + } satisfies WaveformCanvasProps); + useEffect(() => { if (waveFormCanvas.current) { waveFormCanvas.current.width = props.width; waveFormCanvas.current.height = props.height; drawWaveform(waveFormCanvas.current, realProps); } - }, [props, waveFormCanvas]); + }, [watchedProps, realProps.leftPadding, realProps.rightSamples, waveFormCanvas]); return ; }; From 4c93f9117cf34b82de9e518040839365b917183f Mon Sep 17 00:00:00 2001 From: Danielku15 Date: Fri, 23 May 2025 00:21:02 +0200 Subject: [PATCH 6/6] fix: restore correct edit behavior --- src/components/AlphaTabPlayground/sync-point-info.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/AlphaTabPlayground/sync-point-info.ts b/src/components/AlphaTabPlayground/sync-point-info.ts index 4719f87c1..f9a34ea61 100644 --- a/src/components/AlphaTabPlayground/sync-point-info.ts +++ b/src/components/AlphaTabPlayground/sync-point-info.ts @@ -529,7 +529,7 @@ export function updateSyncPointsAfterModification( const newBpmAfter = syncedDuration > 0 ? (synthDuration / syncedDuration) * modifiedMarker.synthBpm : 0; modifiedMarker.modifiedTempo = newBpmAfter; - const tickDuration = nextMarker.synthTick - modifiedMarker.synthTick; + const tickDuration = s.syncPointMarkers[modifiedIndex + 1].synthTick - modifiedMarker.synthTick; let syncedTimePosition = modifiedMarker.syncTime + ticksToMilliseconds(tickDuration, newBpmAfter); for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) {