diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml new file mode 100644 index 000000000..8d32418d5 --- /dev/null +++ b/.github/workflows/ios.yml @@ -0,0 +1,287 @@ +name: iOS pipeline + +on: + push: + branches: + - main + paths: + - interfaces/*/src/** + - interfaces/*/examples/** + - interfaces/*/Cargo.toml + - models/*/src/** + - models/*/examples/** + - models/*/Cargo.toml + - floneum/*/src/** + - floneum/*/examples/** + - floneum/*/Cargo.toml + - fusor-ml/*/src/** + - fusor-ml/*/examples/** + - fusor-ml/*/Cargo.toml + - src/** + - .github/** + - Cargo.toml + + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: + - main + paths: + - interfaces/*/src/** + - interfaces/*/examples/** + - interfaces/*/Cargo.toml + - models/*/src/** + - models/*/examples/** + - models/*/Cargo.toml + - floneum/*/src/** + - floneum/*/examples/** + - floneum/*/Cargo.toml + - fusor-ml/*/src/** + - fusor-ml/*/examples/** + - fusor-ml/*/Cargo.toml + - src/** + - .github/** + - Cargo.toml + +# workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + name: iOS build + runs-on: macos-latest + + env: + TOOLCHAIN: nightly + CARGO_OPTIMIZATION_FLAGS: "-C debuginfo=0 -C lto=off -C embed-bitcode=no -C opt-level=z -C codegen-units=1 -C strip=symbols -C link-arg=-Wl,-dead_strip -C link-arg=-Wl,--gc-sections" + strategy: + fail-fast: false + matrix: + platform: + - { + id: "ios", + name: "iOS", + sdk: iphoneos, + arch: arm64, + base_arch: ios, + openssl_prefix: "_build/ios-openssl/build/openssl.ios", + openssl_arch: "ios64-xcrun", + otp_arch: "aarch64-apple-ios", + rust_target: "aarch64-apple-ios", + } + - { + id: "ios_sim_arm64", + name: "iOS Simulator arm64", + sdk: iphonesimulator, + arch: arm64, + base_arch: ios_sim, + openssl_prefix: "_build/ios-openssl/build/openssl.iossim.arm64", + openssl_arch: "iossimulator-arm64-xcrun", + otp_arch: "aarch64-apple-iossimulator", + rust_target: "aarch64-apple-ios-sim", + } + - { + id: "ios_sim_x86_64", + name: "iOS Simulator x86_64", + sdk: iphonesimulator, + arch: x86_64, + base_arch: ios_sim, + openssl_prefix: "_build/ios-openssl/build/openssl.iossim.x86_64", + openssl_arch: "iossimulator-x86_64-xcrun", + otp_arch: "x86_64-apple-iossimulator", + rust_target: "x86_64-apple-ios", + } + + # - { + # id: 'watchos', + # name: 'watchOS', + # sdk: watchos, + # arch: arm64, + # base_arch: watchos, + # openssl_prefix: '_build/ios-openssl/build/openssl.watchos', + # #openssl_arch: 'watchos-arm64_32-xcrun', + # openssl_arch: 'watchos-arm64-xcrun', + # otp_arch: 'aarch64-apple-watchos', + # rust_target: 'aarch64-apple-watchos', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + # - { + # id: 'watchos_sim', + # name: 'watchOS Simulator', + # sdk: watchsimulator, + # arch: x86_64, + # base_arch: watchos_sim, + # openssl_prefix: '_build/ios-openssl/build/openssl.watchossim.x86_64', + # openssl_arch: 'watchsimulator-xcrun', + # otp_arch: 'x86_64-apple-watchossim', + # rust_target: 'x86_64-apple-watchos-sim', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + # - { + # id: 'watchos_sim', + # name: 'watchOS Simulator', + # sdk: watchsimulator, + # arch: arm64, + # base_arch: watchos_sim, + # openssl_prefix: '_build/ios-openssl/build/openssl.watchossim.arm64', + # openssl_arch: 'watchsimulator-xcrun', + # otp_arch: 'aarch64-apple-watchossim', + # rust_target: 'aarch64-apple-watchos-sim', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + # - { + # id: 'tvos', + # name: 'tvOS', + # sdk: appletvos, + # arch: arm64, + # base_arch: tvos, + # openssl_prefix: '_build/ios-openssl/build/openssl.tvos', + # openssl_arch: 'tvos-xcrun', + # otp_arch: 'aarch64-apple-tvos', + # rust_target: 'aarch64-apple-tvos', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + # - { + # id: 'tvos_sim_arm64', + # name: 'tvOS Simulator arm64', + # sdk: appletvsimulator, + # arch: arm64, + # base_arch: tvos_sim, + # openssl_prefix: '_build/ios-openssl/build/openssl.tvossim.arm64', + # openssl_arch: 'tvossimulator-xcrun', + # otp_arch: 'aarch64-apple-tvossimulator', + # rust_target: 'aarch64-apple-tvos-sim', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + # - { + # id: 'tvos_sim_x86_64', + # name: 'tvOS Simulator x86_64', + # sdk: appletvsimulator, + # arch: x86_64, + # base_arch: tvos_sim, + # openssl_prefix: '_build/ios-openssl/build/openssl.tvossim.x86_64', + # openssl_arch: 'tvossimulator-x86_64-xcrun', + # otp_arch: 'x86_64-apple-tvos', + # rust_target: 'x86_64-apple-tvos', + # rust_is_3rd_tier: true, + # erl-opts: ' --disable-hipe --disable-sctp --disable-vm-probes --without-termcap --without-odbc --disable-jit', + # } + + # - { + # id: "visionos", + # name: "visionOS", + # sdk: xros, + # arch: arm64, + # base_arch: xros, + # openssl_prefix: "_build/ios-openssl/build/openssl.xros", + # openssl_arch: "xros-xcrun", + # otp_arch: "aarch64-apple-visionos", + # rust_target: "aarch64-apple-visionos", + # rust_is_3rd_tier: true, + # } + # - { + # id: "visionos_sim_arm64", + # name: "visionOS Simulator arm64", + # sdk: xrsimulator, + # arch: arm64, + # base_arch: xrossim, + # openssl_prefix: "_build/ios-openssl/build/openssl.xrossim.arm64", + # openssl_arch: "xrossimulator-xcrun", + # #otp_arch: 'aarch64-apple-visionos-sim', + # otp_arch: "aarch64-apple-visionossim", + # rust_target: "aarch64-apple-visionos-sim", + # rust_is_3rd_tier: true, + # } + # - { + # id: 'visionos_sim_x86_64', + # name: 'visionOS Simulator x86_64', + # sdk: xrsimulator, + # arch: x86_64, + # base_arch: xrossim, + # openssl_prefix: '_build/ios-openssl/build/openssl.xrossim.x86_64', + # openssl_arch: 'xrossimulator-xcrun', + # otp_arch: 'x86_64-apple-visionos', + # rust_target: 'x86_64-apple-visionos', + # rust_is_3rd_tier: true, + # } + # uses: actions/checkout@v4 + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.TOOLCHAIN }} + targets: ${{ matrix.platform.rust_target }} + # llvm-tools-preview stuff is essential, breaks the build otherwise + components: rust-src, llvm-tools-preview + - name: Build + run: | + #rustup target add ${{matrix.platform.rust_target}} + # --features rustler/staticlib + # cargo +nightly rustc -Zbuild-std --lib --crate-type staticlib --target=${{matrix.platform.rust_target}} -- $CARGO_OPTIMIZATION_FLAGS + # requires onnxruntime kalosm/sound, + cargo +nightly -Zbuild-std test --target ${{matrix.platform.rust_target}} --all --tests --features kalosm/language,kalosm/sound,kalosm/vision,kalosm/remote,kalosm/scrape,kalosm/metal --exclude fusor-ml-core + find . -name "*.a" -exec ls -lah {} \; + find . -name "*.a" -exec lipo -info {} \; + + # test: + # # if: false + # # && github.event.pull_request.draft == false + # name: iOS Test Suite + # runs-on: macos-latest + + # strategy: + # matrix: + # platform: + # - { + # id: "ios", + # name: "iOS", + # sdk: iphoneos, + # arch: arm64, + # base_arch: ios, + # openssl_prefix: "_build/ios-openssl/build/openssl.ios", + # openssl_arch: "ios64-xcrun", + # otp_arch: "aarch64-apple-ios", + # rust_target: "aarch64-apple-ios", + # } + # - { + # id: "ios_sim_arm64", + # name: "iOS Simulator arm64", + # sdk: iphonesimulator, + # arch: arm64, + # base_arch: ios_sim, + # openssl_prefix: "_build/ios-openssl/build/openssl.iossim.arm64", + # openssl_arch: "iossimulator-arm64-xcrun", + # otp_arch: "aarch64-apple-iossimulator", + # rust_target: "aarch64-apple-ios-sim", + # } + # - { + # id: "ios_sim_x86_64", + # name: "iOS Simulator x86_64", + # sdk: iphonesimulator, + # arch: x86_64, + # base_arch: ios_sim, + # openssl_prefix: "_build/ios-openssl/build/openssl.iossim.x86_64", + # openssl_arch: "iossimulator-x86_64-xcrun", + # otp_arch: "x86_64-apple-iossimulator", + # rust_target: "x86_64-apple-ios", + # } + # environment: + # name: OPENAI_API_KEY + # steps: + # - uses: dtolnay/rust-toolchain@stable + # - uses: Swatinem/rust-cache@v2 + # - uses: actions/checkout@v4 + # - name: Run tests + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # # kalosm/remote, + # run: cargo test --target ${{matrix.platform.rust_target}} --all --tests --features kalosm/language,kalosm/sound,kalosm/vision,kalosm/scrape,kalosm/metal --exclude fusor-ml-core diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index b66c715e5..175c7e007 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -49,7 +49,8 @@ concurrency: jobs: test: - if: github.event.pull_request.draft == false + if: false + #if: github.event.pull_request.draft == false name: Mac Test Suite runs-on: macos-latest environment: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml.txt similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/main.yml.txt diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml.txt similarity index 100% rename from .github/workflows/nightly.yml rename to .github/workflows/nightly.yml.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml.txt similarity index 100% rename from .github/workflows/windows.yml rename to .github/workflows/windows.yml.txt diff --git a/interfaces/kalosm-sound/Cargo.toml b/interfaces/kalosm-sound/Cargo.toml index 989a0f61f..f07aeef3a 100644 --- a/interfaces/kalosm-sound/Cargo.toml +++ b/interfaces/kalosm-sound/Cargo.toml @@ -34,7 +34,7 @@ ort-sys = { version = "=2.0.0-rc.4", optional = true } nnnoiseless = { version = "0.5.1", optional = true } [features] -default = ["voice_detection", "denoise"] +default = ["denoise"] metal = ["candle-core/metal", "rwhisper/accelerate", "rwhisper/metal"] cuda = ["candle-core/cuda", "rwhisper/cuda", "rwhisper/cudnn"] mkl = ["candle-core/mkl", "rwhisper/mkl"] diff --git a/interfaces/kalosm-sound/examples/transcribe.rs b/interfaces/kalosm-sound/examples/transcribe.rs index 40798aa48..d6a547dc9 100644 --- a/interfaces/kalosm-sound/examples/transcribe.rs +++ b/interfaces/kalosm-sound/examples/transcribe.rs @@ -14,7 +14,7 @@ async fn main() -> Result<(), anyhow::Error> { let stream = mic.stream(); // Chunk that audio into chunks based on voice activity - let vad = stream.voice_activity_stream().rechunk_voice_activity(); + let vad = stream.denoise_and_detect_voice_activity().rechunk_voice_activity(); // And then transcribe the audio into text let mut text_stream = vad.transcribe(model); diff --git a/interfaces/kalosm-sound/examples/voice_activity_detection.rs b/interfaces/kalosm-sound/examples/voice_activity_detection.rs index 4ccae1bf8..09b771422 100644 --- a/interfaces/kalosm-sound/examples/voice_activity_detection.rs +++ b/interfaces/kalosm-sound/examples/voice_activity_detection.rs @@ -6,7 +6,7 @@ use rodio::OutputStream; async fn main() -> Result<(), anyhow::Error> { let mic_input = MicInput::default(); let stream = mic_input.stream(); - let mut vad = stream.voice_activity_stream(); + let mut vad = stream.denoise_and_detect_voice_activity(); let (_device, stream_handle) = OutputStream::try_default().unwrap(); let sink = rodio::Sink::try_new(&stream_handle)?; while let Some(VoiceActivityDetectorOutput { diff --git a/interfaces/kalosm-sound/src/transform/mod.rs b/interfaces/kalosm-sound/src/transform/mod.rs index 258679828..bb43b6dd6 100644 --- a/interfaces/kalosm-sound/src/transform/mod.rs +++ b/interfaces/kalosm-sound/src/transform/mod.rs @@ -13,7 +13,7 @@ mod voice_audio_detector_ext; #[cfg(any(feature = "voice_detection", feature = "denoise"))] pub use voice_audio_detector_ext::*; -#[cfg(feature = "voice_detection")] +#[cfg(feature = "denoise")] mod transcribe; -#[cfg(feature = "voice_detection")] +#[cfg(feature = "denoise")] pub use transcribe::*; diff --git a/interfaces/kalosm-sound/src/transform/transcribe.rs b/interfaces/kalosm-sound/src/transform/transcribe.rs index c2a4074d1..3e7aa52c6 100644 --- a/interfaces/kalosm-sound/src/transform/transcribe.rs +++ b/interfaces/kalosm-sound/src/transform/transcribe.rs @@ -1,6 +1,6 @@ use rwhisper::ChunkedTranscriptionTask; -use super::voice_audio_detector::*; +use super::denoise::*; use super::voice_audio_detector_ext::*; use crate::AsyncSource; @@ -32,10 +32,11 @@ pub trait AsyncSourceTranscribeExt: AsyncSource + Unpin + Send + Sized + 'static fn transcribe( self, model: rwhisper::Whisper, - ) -> ChunkedTranscriptionTask>> + ) -> ChunkedTranscriptionTask>> { rwhisper::TranscribeChunkedAudioStreamExt::transcribe( - self.voice_activity_stream().rechunk_voice_activity(), + self.denoise_and_detect_voice_activity() + .rechunk_voice_activity(), model, ) } diff --git a/interfaces/kalosm/Cargo.toml b/interfaces/kalosm/Cargo.toml index cf4692843..f5ee73a05 100644 --- a/interfaces/kalosm/Cargo.toml +++ b/interfaces/kalosm/Cargo.toml @@ -74,7 +74,6 @@ tokio = { version = "1", features = ["full"] } [dev-dependencies.kalosm] features = [ - "sound", "language", "vision", "remote", diff --git a/interfaces/kalosm/examples/live-qa.rs b/interfaces/kalosm/examples/live-qa.rs index 8d13db5ab..216297efc 100644 --- a/interfaces/kalosm/examples/live-qa.rs +++ b/interfaces/kalosm/examples/live-qa.rs @@ -33,7 +33,7 @@ async fn main() -> Result<(), anyhow::Error> { // Chunk the audio into chunks based on voice activity let mut audio_chunks = mic_input .stream() - .voice_activity_stream() + .denoise_and_detect_voice_activity() .rechunk_voice_activity(); while let Some(input) = audio_chunks.next().await { let mut transcribed = model.transcribe(input); diff --git a/out.txt b/out.txt new file mode 100644 index 000000000..e69de29bb