Feature/stem audio to midi transcription

CHANGELOG.md

@@ -8,6 +8,7 @@ development artifact builds use `vMAJOR.MINOR.PATCH-dev.N`.
 
 ## Unreleased
 
+- Added fully offline per-stem Audio-to-MIDI transcription with a bundled Basic Pitch model, native C++ inference, cancellable track progress, polyphonic Notation/MIDI notes, and project persistence. Basic Pitch is unavailable for Drum stems, and re-transcription now warns before replacing existing stem notes and rests.
 - Added inline flat, natural, and sharp signs to Notation with Leland glyphs, one-shot note entry, selected tied-note editing, compact duration and accidental track menus, keyboard shortcuts, persistence, and MusicXML export.
 - Added automatic rhythmic beaming for eighth and sixteenth notes in supported simple and compound Notation meters, including shared stem direction, sloped beams, secondary beam breaks, and beamlets.
 - Fixed Leland eighth-note and shorter flags separating from their stems in Notation chord and Drum rendering.

JammLab/ContentView.swift

@@ -65,9 +65,11 @@ struct ContentView: View {
         }
         .onChange(of: viewModel.importedFile?.sourceMediaURL) { _, _ in
             stemMIDIPageStartTimes.removeAll()
+            viewModel.cancelPendingStemTranscriptionOverwrite()
         }
         .onChange(of: viewModel.currentProjectURL) { _, _ in
             stemMIDIPageStartTimes.removeAll()
+            viewModel.cancelPendingStemTranscriptionOverwrite()
         }
         .sheet(isPresented: $isEditingMarker) {
             RenameNoteDialog(
@@ -92,6 +94,20 @@ struct ContentView: View {
                 viewModel.clearError()
             }
         }
+        .confirmationDialog(
+            "Replace \(viewModel.pendingStemTranscriptionOverwrite?.stemType.title ?? "stem") notation?",
+            isPresented: pendingStemTranscriptionOverwriteBinding,
+            titleVisibility: .visible
+        ) {
+            Button("Replace Existing Notes", role: .destructive) {
+                viewModel.confirmPendingStemTranscriptionOverwrite()
+            }
+            Button("Cancel", role: .cancel) {
+                viewModel.cancelPendingStemTranscriptionOverwrite()
+            }
+        } message: {
+            Text("All existing notes and rests on this stem will be deleted and replaced by the new transcription.")
+        }
     }
 
     private var errorAlertBinding: Binding<Bool> {
@@ -105,6 +121,17 @@ struct ContentView: View {
         )
     }
 
+    private var pendingStemTranscriptionOverwriteBinding: Binding<Bool> {
+        Binding(
+            get: { viewModel.pendingStemTranscriptionOverwrite != nil },
+            set: { isPresented in
+                if !isPresented {
+                    viewModel.cancelPendingStemTranscriptionOverwrite()
+                }
+            }
+        )
+    }
+
     private func cancelMarkerEditing() {
         isEditingMarker = false
         editingMarkerID = nil

JammLab/JammLab-Bridging-Header.h

@@ -0,0 +1 @@
+#import "Transcription/Native/JMBasicPitchBridge.h"

JammLab/Models/JammLabProject.swift

@@ -24,6 +24,7 @@ struct JammLabProject: Codable {
     var notes: [TimecodedNote]
     var harmonySymbols: [HarmonySymbol]
     var notationItems: [NotationMeasureItem]
+    var stemTranscriptionTracks: [StemTranscriptionTrack]
     var notationPartClefs: [NotationPartID: Clef]
     var projectKeySelection: ProjectKeySelection?
     var loopStart: TimeInterval
@@ -48,7 +49,7 @@ struct JammLabProject: Codable {
     var visibleNotationPartIDs: Set<NotationPartID>
 
     init(
-        formatVersion: Int = 14,
+        formatVersion: Int = 15,
         audioBookmarkData: Data,
         artifactRootBookmarkData: Data? = nil,
         audioDisplayName: String,
@@ -57,6 +58,7 @@ struct JammLabProject: Codable {
         notes: [TimecodedNote],
         harmonySymbols: [HarmonySymbol] = [],
         notationItems: [NotationMeasureItem] = [],
+        stemTranscriptionTracks: [StemTranscriptionTrack] = [],
         notationPartClefs: [NotationPartID: Clef] = [:],
         projectKeySelection: ProjectKeySelection? = nil,
         loopStart: TimeInterval,
@@ -89,6 +91,7 @@ struct JammLabProject: Codable {
         self.notes = notes
         self.harmonySymbols = harmonySymbols
         self.notationItems = notationItems
+        self.stemTranscriptionTracks = stemTranscriptionTracks
         self.notationPartClefs = notationPartClefs
         self.projectKeySelection = projectKeySelection
         self.loopStart = loopStart
@@ -123,6 +126,7 @@ struct JammLabProject: Codable {
         case notes
         case harmonySymbols
         case notationItems
+        case stemTranscriptionTracks
         case notationPartClefs
         case projectKeySelection
         case loopStart
@@ -158,6 +162,10 @@ struct JammLabProject: Codable {
         notes = try container.decode([TimecodedNote].self, forKey: .notes)
         harmonySymbols = try container.decodeIfPresent([HarmonySymbol].self, forKey: .harmonySymbols) ?? []
         notationItems = try container.decodeIfPresent([NotationMeasureItem].self, forKey: .notationItems) ?? []
+        stemTranscriptionTracks = try container.decodeIfPresent(
+            [StemTranscriptionTrack].self,
+            forKey: .stemTranscriptionTracks
+        ) ?? []
         notationPartClefs = try container.decodeIfPresent([NotationPartID: Clef].self, forKey: .notationPartClefs) ?? [:]
         projectKeySelection = try container.decodeIfPresent(ProjectKeySelection.self, forKey: .projectKeySelection)
         loopStart = try container.decode(TimeInterval.self, forKey: .loopStart)
@@ -195,6 +203,7 @@ struct JammLabProject: Codable {
         try container.encode(notes, forKey: .notes)
         try container.encode(harmonySymbols, forKey: .harmonySymbols)
         try container.encode(notationItems, forKey: .notationItems)
+        try container.encode(stemTranscriptionTracks, forKey: .stemTranscriptionTracks)
         try container.encode(notationPartClefs, forKey: .notationPartClefs)
         try container.encodeIfPresent(projectKeySelection, forKey: .projectKeySelection)
         try container.encode(loopStart, forKey: .loopStart)

JammLab/Models/NotationScoreModels.swift

@@ -170,9 +170,18 @@ struct NotationPartID: Codable, Hashable, Identifiable, Equatable {
         NotationPartID(rawValue: "stem:\(type.rawValue)")
     }
 
+    static func stemTranscription(_ type: StemType, trackID: UUID) -> NotationPartID {
+        NotationPartID(rawValue: "transcription:\(type.rawValue):\(trackID.uuidString.lowercased())")
+    }
+
     var stemType: StemType? {
-        guard rawValue.hasPrefix("stem:") else { return nil }
-        return StemType(rawValue: String(rawValue.dropFirst("stem:".count)))
+        if rawValue.hasPrefix("stem:") {
+            return StemType(rawValue: String(rawValue.dropFirst("stem:".count)))
+        }
+        guard rawValue.hasPrefix("transcription:") else { return nil }
+        let components = rawValue.split(separator: ":", omittingEmptySubsequences: false)
+        guard components.count == 3 else { return nil }
+        return StemType(rawValue: String(components[1]))
     }
 
     var isMain: Bool {
@@ -245,6 +254,21 @@ struct NotationPartDescriptor: Equatable, Identifiable {
         }
     }
 
+    static func stemTranscription(
+        _ type: StemType,
+        id: NotationPartID,
+        sequence: Int
+    ) -> NotationPartDescriptor {
+        let base = stem(type)
+        return NotationPartDescriptor(
+            id: id,
+            title: "\(base.title) Transcription \(sequence)",
+            abbreviation: "\(base.abbreviation) T\(sequence)",
+            instrumentName: base.instrumentName,
+            instrumentSound: base.instrumentSound
+        )
+    }
+
     private static func make(
         id: NotationPartID,
         title: String,

JammLab/Models/ProjectEditableState.swift

@@ -4,6 +4,7 @@ struct ProjectEditableState: Equatable {
     var notes: [TimecodedNote]
     var harmonySymbols: [HarmonySymbol] = []
     var notationItems: [NotationMeasureItem] = []
+    var stemTranscriptionTracks: [StemTranscriptionTrack] = []
     var notationPartClefs: [NotationPartID: Clef] = [:]
     var visibleNotationPartIDs: Set<NotationPartID> = [.main]
     var projectKeySelection: ProjectKeySelection? = nil
@@ -28,6 +29,7 @@ struct ProjectPersistedEditableState: Equatable {
     var notes: [TimecodedNote]
     var harmonySymbols: [HarmonySymbol] = []
     var notationItems: [NotationMeasureItem] = []
+    var stemTranscriptionTracks: [StemTranscriptionTrack] = []
     var notationPartClefs: [NotationPartID: Clef] = [:]
     var stemNotationTrackCollapsed: [StemType: Bool] = [:]
     var stemNoteDisplayModes: [StemType: StemNoteDisplayMode] = [:]

JammLab/Models/ProjectStateNormalizer.swift

@@ -184,6 +184,92 @@ struct ProjectStateNormalizer {
             }
     }
 
+    static func normalizedStemTranscriptionTracks(
+        _ tracks: [StemTranscriptionTrack],
+        duration: TimeInterval,
+        notationItems: [NotationMeasureItem]
+    ) -> [StemTranscriptionTrack] {
+        let duration = normalizedDuration(duration)
+        let notationItemsByID = Dictionary(
+            notationItems.map { ($0.id, $0) },
+            uniquingKeysWith: { first, _ in first }
+        )
+        var seenTrackIDs = Set<UUID>()
+
+        return tracks.compactMap { track in
+            guard seenTrackIDs.insert(track.id).inserted,
+                  !track.sourceFingerprint.path.isEmpty,
+                  track.sourceFingerprint.fileSize >= 0,
+                  track.sourceFingerprint.modificationTime.isFinite
+            else {
+                return nil
+            }
+
+            let partID = track.notationPartID.stemType == track.stemType
+                ? track.notationPartID
+                : .stem(track.stemType)
+            let notes = track.notes.compactMap { note -> StemTranscriptionNote? in
+                guard (0...127).contains(note.midiPitch),
+                      note.rawStartTimeSeconds.isFinite,
+                      note.rawEndTimeSeconds.isFinite,
+                      note.rawStartTimeSeconds >= 0,
+                      note.rawEndTimeSeconds > note.rawStartTimeSeconds,
+                      note.projectStartTimeSeconds.isFinite,
+                      note.projectEndTimeSeconds.isFinite,
+                      note.projectEndTimeSeconds > note.projectStartTimeSeconds,
+                      note.confidence.isFinite
+                else {
+                    return nil
+                }
+                let projectStart = min(duration, max(0, note.projectStartTimeSeconds))
+                let projectEnd = min(duration, max(0, note.projectEndTimeSeconds))
+                guard projectEnd > projectStart else { return nil }
+
+                return StemTranscriptionNote(
+                    id: note.id,
+                    midiPitch: note.midiPitch,
+                    rawStartTimeSeconds: note.rawStartTimeSeconds,
+                    rawEndTimeSeconds: note.rawEndTimeSeconds,
+                    projectStartTimeSeconds: projectStart,
+                    projectEndTimeSeconds: projectEnd,
+                    confidence: min(1, max(0, note.confidence)),
+                    pitchBends: note.pitchBends,
+                    notationItemIDs: note.notationItemIDs.filter {
+                        notationItemsByID[$0]?.partID == partID
+                    }
+                )
+            }
+            let timings = track.timings.flatMap(normalizedTranscriptionTimings)
+            return StemTranscriptionTrack(
+                id: track.id,
+                stemType: track.stemType,
+                notationPartID: partID,
+                sourceFingerprint: track.sourceFingerprint,
+                createdAt: track.createdAt,
+                configuration: track.configuration,
+                notes: notes,
+                timings: timings,
+                warnings: track.warnings
+            )
+        }
+        .sorted { $0.createdAt < $1.createdAt }
+    }
+
+    private static func normalizedTranscriptionTimings(
+        _ timings: StemTranscriptionTimings
+    ) -> StemTranscriptionTimings? {
+        let values = [
+            timings.audioPreparationSeconds,
+            timings.modelLoadSeconds,
+            timings.inferenceSeconds,
+            timings.postProcessingSeconds,
+            timings.totalSeconds,
+            timings.processedDurationSeconds
+        ]
+        guard values.allSatisfy({ $0.isFinite && $0 >= 0 }) else { return nil }
+        return timings
+    }
+
     private static func normalizedTieTargetItemID(
         for item: NotationMeasureItem,
         availableItemsByID: [String: NotationMeasureItem]

JammLab/Models/StemModels.swift

@@ -14,6 +14,7 @@ enum PlaybackMode: String, Codable, CaseIterable, Identifiable {
             return "Stems"
         }
     }
+
 }
 
 enum StemType: String, Codable, CaseIterable, Identifiable {
@@ -46,6 +47,13 @@ enum StemType: String, Codable, CaseIterable, Identifiable {
         }
     }
 
+    /// Spotify Basic Pitch is intended for pitched audio, not percussion.
+    /// Keep this policy centralized so UI and application/service boundaries
+    /// cannot accidentally diverge.
+    var supportsBasicPitchTranscription: Bool {
+        self != .drums
+    }
+
     var canonicalStemFilename: String {
         "\(rawValue).wav"
     }