From 268e42de6061d66df21e3526985872d7bd6c5834 Mon Sep 17 00:00:00 2001 From: rockerritesh <076bct088.sumit@pcampus.edu.np> Date: Fri, 29 May 2026 12:08:24 +0545 Subject: [PATCH] feat: IELTS speaking & reading trainer with on-device Whisper ASR Transform the TTS app into an IELTS English trainer (TTS retained as "Voice Studio"). Three tabs via Navigation Compose: Reading, Speaking, Voice Studio. ASR (on-device, offline): - WhisperAsrEngine: quantized whisper-tiny.en ONNX (encoder+decoder), greedy decode over 30s windows, bundled in assets/asr (~41 MB, LFS). - MelSpectrogram (80-bin log-mel) validated bit-exact vs HF feature extractor; byte-level BPE WhisperTokenizer; AudioRecorder (16 kHz). - Shared via AsrEngineProvider across both scored features. Reading practice: - 200 graded passages (bands 4.5-9, 15 topics) in assets/ielts. - WerScorer: Word Error Rate + word-level alignment/highlighting. - IeltsScorer: read-aloud four-criteria band estimate. - Listen button reuses KittenTTS to read the passage. Free speaking practice: - IELTS prompts (Parts 1-3) in assets/ielts/speaking_prompts.json. - FluencyAnalyzer: pace, long pauses, fillers from the audio. - FreeSpeechScorer: full four-criteria band estimate (Task Response, Fluency & Coherence, Lexical Resource, Grammatical Range) from the candidate's own words, with prompt-coverage feedback. Tooling: - tools/download_whisper_onnx.py: fetch pre-converted quantized model (no PyTorch); tools/export_whisper_onnx.py for optimum<2.0 export. - RECORD_AUDIO permission; navigation-compose dependency. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 134 +- app/build.gradle.kts | 1 + app/src/main/AndroidManifest.xml | 2 + app/src/main/assets/asr/README.md | 60 + app/src/main/assets/asr/asr_config.json | 16 + app/src/main/assets/asr/vocab.json | 50259 ++++++++++++++++ app/src/main/assets/asr/whisper_decoder.onnx | 3 + app/src/main/assets/asr/whisper_encoder.onnx | 3 + app/src/main/assets/ielts/paragraphs.json | 202 + .../main/assets/ielts/speaking_prompts.json | 27 + .../java/com/kittenml/tts/MainActivity.kt | 147 + .../com/kittenml/tts/asr/AsrEngineProvider.kt | 23 + .../java/com/kittenml/tts/asr/AsrState.kt | 12 + .../com/kittenml/tts/asr/AudioRecorder.kt | 124 + .../com/kittenml/tts/asr/MelSpectrogram.kt | 156 + .../com/kittenml/tts/asr/WhisperAsrEngine.kt | 240 + .../com/kittenml/tts/asr/WhisperTokenizer.kt | 66 + .../java/com/kittenml/tts/data/Paragraph.kt | 25 + .../kittenml/tts/data/ParagraphRepository.kt | 37 + .../com/kittenml/tts/data/SpeakingPrompt.kt | 30 + .../tts/data/SpeakingPromptRepository.kt | 29 + .../java/com/kittenml/tts/scoring/BandUtil.kt | 15 + .../kittenml/tts/scoring/FluencyAnalyzer.kt | 82 + .../kittenml/tts/scoring/FreeSpeechScorer.kt | 128 + .../kittenml/tts/scoring/IeltsAssessment.kt | 25 + .../com/kittenml/tts/scoring/IeltsScorer.kt | 81 + .../com/kittenml/tts/scoring/ScoreResult.kt | 46 + .../com/kittenml/tts/scoring/WerScorer.kt | 95 + .../ui/screen/practice/PracticeListScreen.kt | 143 + .../tts/ui/screen/practice/PracticeScreen.kt | 386 + .../ui/screen/practice/PracticeViewModel.kt | 150 + .../ui/screen/speaking/SpeakingListScreen.kt | 110 + .../tts/ui/screen/speaking/SpeakingScreen.kt | 322 + .../ui/screen/speaking/SpeakingViewModel.kt | 119 + gradle/libs.versions.toml | 2 + tools/download_whisper_onnx.py | 92 + tools/export_whisper_onnx.py | 108 + 37 files changed, 53485 insertions(+), 15 deletions(-) create mode 100644 app/src/main/assets/asr/README.md create mode 100644 app/src/main/assets/asr/asr_config.json create mode 100644 app/src/main/assets/asr/vocab.json create mode 100644 app/src/main/assets/asr/whisper_decoder.onnx create mode 100644 app/src/main/assets/asr/whisper_encoder.onnx create mode 100644 app/src/main/assets/ielts/paragraphs.json create mode 100644 app/src/main/assets/ielts/speaking_prompts.json create mode 100644 app/src/main/java/com/kittenml/tts/asr/AsrEngineProvider.kt create mode 100644 app/src/main/java/com/kittenml/tts/asr/AsrState.kt create mode 100644 app/src/main/java/com/kittenml/tts/asr/AudioRecorder.kt create mode 100644 app/src/main/java/com/kittenml/tts/asr/MelSpectrogram.kt create mode 100644 app/src/main/java/com/kittenml/tts/asr/WhisperAsrEngine.kt create mode 100644 app/src/main/java/com/kittenml/tts/asr/WhisperTokenizer.kt create mode 100644 app/src/main/java/com/kittenml/tts/data/Paragraph.kt create mode 100644 app/src/main/java/com/kittenml/tts/data/ParagraphRepository.kt create mode 100644 app/src/main/java/com/kittenml/tts/data/SpeakingPrompt.kt create mode 100644 app/src/main/java/com/kittenml/tts/data/SpeakingPromptRepository.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/BandUtil.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/FluencyAnalyzer.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/FreeSpeechScorer.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/IeltsAssessment.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/IeltsScorer.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/ScoreResult.kt create mode 100644 app/src/main/java/com/kittenml/tts/scoring/WerScorer.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/practice/PracticeListScreen.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/practice/PracticeScreen.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/practice/PracticeViewModel.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/speaking/SpeakingListScreen.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/speaking/SpeakingScreen.kt create mode 100644 app/src/main/java/com/kittenml/tts/ui/screen/speaking/SpeakingViewModel.kt create mode 100644 tools/download_whisper_onnx.py create mode 100644 tools/export_whisper_onnx.py diff --git a/README.md b/README.md index 7c7789f..567f841 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,34 @@ -# KittenTTS Android - -On-device text-to-speech Android app powered by [KittenML](https://github.com/KittenML) neural TTS models. Runs entirely offline — no internet required after install. - -Ported from the [iOS version](https://github.com/user/kitten-tts-ios) (Swift/SwiftUI) to Kotlin/Jetpack Compose. +# KittenTTS Android — IELTS Speaking & Reading Trainer + +An on-device English learning app for **IELTS speaking and reading practice**, built on +[KittenML](https://github.com/KittenML) neural TTS plus an on-device **Whisper** speech +recognizer. Read a passage aloud, and the app transcribes your voice and scores your +**reading accuracy (Word Error Rate)** and an **IELTS band estimate** across the four +official criteria — all fully offline, no internet required after install. + +It also keeps the original **Voice Studio** (text-to-speech) so you can hear a model read +any passage before you try it yourself. + +## How it works + +The app has three tabs: **Reading**, **Speaking**, and **Voice Studio**. + +**Reading (read-aloud)** +1. Pick from 200 bundled passages, graded across IELTS bands and topics. +2. **Listen** — a neural voice reads the passage aloud (KittenTTS). +3. **Record & read aloud** — your microphone audio is captured at 16 kHz. +4. **Score** — Whisper-tiny.en (ONNX, on-device) transcribes your speech; it is aligned + word-by-word against the passage to compute Word Error Rate, reading accuracy, and an + IELTS-style band breakdown. + +**Speaking (free response)** +1. Pick an IELTS-style prompt (Part 1 questions, Part 2 cue cards, Part 3 discussion). +2. **Record your answer** in your own words. +3. **Score** — Whisper transcribes it, then a fluency analyzer (pace, pauses, fillers) and + a free-speech scorer estimate all four official criteria — now genuinely from *your own* + vocabulary and grammar: **Task Response · Fluency & Coherence · Lexical Resource · + Grammatical Range & Accuracy**, plus an overall band. It also shows which prompt points + you covered and a transcript. ## Screenshots @@ -14,6 +40,21 @@ Ported from the [iOS version](https://github.com/user/kitten-tts-ios) (Swift/Swi ## Features +**IELTS reading practice** +- 200 bundled read-aloud passages graded by IELTS band (4.5–9) and topic +- On-device speech recognition (Whisper-tiny.en, ONNX) — fully offline +- Word Error Rate scoring with word-level highlighting (correct / misread / skipped) +- IELTS band estimate across the four official criteria +- "Listen" button — hear a neural voice read the passage first + +**IELTS speaking practice (free response)** +- IELTS-style prompts across Part 1, Part 2 (cue cards), and Part 3 +- Answer in your own words; Whisper transcribes on-device +- Fluency analysis from the audio: words-per-minute, long pauses, filler words +- Full four-criteria band estimate from your own vocabulary and grammar +- Prompt-point coverage feedback + transcript + +**Voice Studio (text-to-speech)** - 3 model sizes: **Nano** (15M), **Micro** (40M), **Mini** (80M) - 8 voices: Rosie, Bella, Jasper, Luna, Bruno, Hugo, Kiki, Leo - Adjustable speed (0.5x – 2.0x) @@ -24,6 +65,31 @@ Ported from the [iOS version](https://github.com/user/kitten-tts-ios) (Swift/Swi ## Architecture +### Speech recognition + scoring (IELTS practice) + +``` +Microphone (16 kHz mono PCM) + → Log-mel spectrogram (80 bins, Whisper spec) + → Whisper encoder ONNX → hidden states + → Whisper decoder ONNX → greedy token decode (30 s windows) + → Byte-level BPE detokenize → transcript + → WER alignment vs. passage → accuracy + highlighting + → IELTS four-criteria band estimate +``` + +The Whisper ONNX models (quantized whisper-tiny.en, ~41 MB) are **bundled in the APK** +(`assets/asr/`) and run on the same ONNX Runtime used for TTS. Fetch them with +[`tools/download_whisper_onnx.py`](tools/download_whisper_onnx.py) (no PyTorch needed) — +see [`app/src/main/assets/asr/README.md`](app/src/main/assets/asr/README.md). The pipeline +(mel spectrogram + greedy decode + byte-level tokenizer) is validated to match the +reference HuggingFace implementation exactly. + +> The IELTS band estimate is an honest *approximation* derived from reading accuracy, +> coverage, and pace. A read-aloud task cannot fully measure free-speech lexical/grammatical +> range; those proxies are documented in `IeltsScorer.kt`. + +### Text-to-speech (Voice Studio) + ``` Text Input (any length) → Auto-chunking (max 400 chars at sentence boundaries) @@ -39,10 +105,12 @@ Text Input (any length) | Component | Technology | |-----------|-----------| -| UI | Kotlin + Jetpack Compose | -| ML Inference | ONNX Runtime Android | +| UI | Kotlin + Jetpack Compose + Navigation Compose | +| ML Inference | ONNX Runtime Android (TTS + Whisper ASR) | +| Speech recognition | Whisper-tiny.en (ONNX, on-device) | | Phonemization | espeak-ng (C via JNI/NDK) | -| Audio | AudioTrack (24kHz Float32 PCM) | +| Audio | AudioTrack playback (24kHz) · AudioRecord capture (16kHz) | +| Scoring | Word Error Rate (Levenshtein) + IELTS band heuristics | | Build | Gradle KTS, Android NDK, CMake | ## Download @@ -72,7 +140,12 @@ Get the latest APK from [Releases](https://github.com/rockerritesh/kitten-tts-an ./build-espeak-ng.sh ``` -3. Open in Android Studio and build, or: +3. Fetch the Whisper ASR model into `assets/asr/` (needed for speaking/reading scoring): + ```bash + python3 tools/download_whisper_onnx.py + ``` + +4. Open in Android Studio and build, or: ```bash ./gradlew assembleDebug ``` @@ -82,16 +155,35 @@ Get the latest APK from [Releases](https://github.com/rockerritesh/kitten-tts-an ``` app/src/main/ ├── java/com/kittenml/tts/ -│ ├── MainActivity.kt # Entry point -│ ├── engine/ +│ ├── MainActivity.kt # Entry point + bottom-nav (Practice / Voice Studio) +│ ├── engine/ # TTS │ │ ├── KittenTTSEngine.kt # Core TTS pipeline │ │ ├── EspeakBridge.kt # JNI wrapper │ │ └── AudioPlayer.kt # AudioTrack playback +│ ├── asr/ # Speech recognition +│ │ ├── AudioRecorder.kt # 16 kHz mic capture +│ │ ├── MelSpectrogram.kt # 80-bin log-mel features +│ │ ├── WhisperTokenizer.kt # byte-level BPE decode +│ │ ├── WhisperAsrEngine.kt # encoder/decoder ONNX inference +│ │ └── AsrState.kt +│ ├── scoring/ +│ │ ├── WerScorer.kt # Word Error Rate + alignment +│ │ ├── ScoreResult.kt +│ │ ├── IeltsScorer.kt # read-aloud four-criteria estimate +│ │ ├── IeltsAssessment.kt +│ │ ├── BandUtil.kt # quality → IELTS band helpers +│ │ ├── FluencyAnalyzer.kt # pace / pauses / fillers from audio +│ │ └── FreeSpeechScorer.kt # free-response four-criteria estimate +│ ├── data/ +│ │ ├── Paragraph.kt / ParagraphRepository.kt +│ │ └── SpeakingPrompt.kt / SpeakingPromptRepository.kt │ ├── ui/ │ │ ├── theme/ # Dark theme (Color, Theme, Type) │ │ └── screen/ -│ │ ├── TTSScreen.kt # Main UI -│ │ └── TTSViewModel.kt # State management +│ │ ├── TTSScreen.kt # Voice Studio UI +│ │ ├── TTSViewModel.kt +│ │ ├── practice/ # reading: list + record/score + ViewModel +│ │ └── speaking/ # free speaking: list + record/score + ViewModel │ └── model/ │ ├── TTSModel.kt # Model enum │ └── EngineState.kt # Engine state @@ -100,11 +192,23 @@ app/src/main/ │ ├── espeak-jni.c # JNI glue layer │ └── CMakeLists.txt # NDK build config └── assets/ - ├── models/ # ONNX model files (~168 MB) + ├── models/ # TTS ONNX model files (~168 MB) ├── voices/ # Voice embedding JSONs (~51 MB) - └── espeak-ng-data/ # Phoneme data files (~1 MB) + ├── espeak-ng-data/ # Phoneme data files (~1 MB) + ├── asr/ # Whisper ONNX + vocab (see asr/README.md) + └── ielts/ + ├── paragraphs.json # 200 read-aloud passages + └── speaking_prompts.json # free-speaking prompts (Parts 1–3) ``` +## IELTS passages + +The bundled passages live in [`app/src/main/assets/ielts/paragraphs.json`](app/src/main/assets/ielts/paragraphs.json), +each with `id`, `title`, `topic`, `band`, and `text`. The app ships with 200 graded +passages across 15 topics and IELTS bands 4.5–9.0. Add or edit passages by changing this +file — no code change is needed, as the list is read at runtime. Free-speaking prompts +live alongside it in `speaking_prompts.json`. + ## License Apache 2.0 diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2f0aa1b..9636c22 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -70,6 +70,7 @@ dependencies { implementation(libs.androidx.compose.ui.tooling.preview) implementation(libs.androidx.lifecycle.viewmodel.compose) implementation(libs.androidx.lifecycle.runtime.compose) + implementation(libs.androidx.navigation.compose) implementation(libs.onnxruntime.android) implementation(libs.gson) debugImplementation(libs.androidx.compose.ui.tooling) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index d229393..9284c32 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -1,6 +1,8 @@ + +