diff --git a/.idea/appInsightsSettings.xml b/.idea/appInsightsSettings.xml new file mode 100644 index 0000000..371f2e2 --- /dev/null +++ b/.idea/appInsightsSettings.xml @@ -0,0 +1,26 @@ + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..7643783 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,123 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/planningMode.xml b/.idea/planningMode.xml new file mode 100644 index 0000000..aa32a55 --- /dev/null +++ b/.idea/planningMode.xml @@ -0,0 +1,11 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 94a25f7..35eb1dd 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 83fc14d..7f996b2 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -131,4 +131,10 @@ dependencies { implementation("com.google.code.gson:gson:2.10.1") implementation("com.github.adaptech-cz.Tesseract4Android:tesseract4android:4.7.0") implementation("com.google.zxing:core:3.5.4") + implementation("com.google.mlkit:text-recognition:16.0.1") + implementation("com.google.mlkit:language-id:17.0.5") + implementation("com.google.mlkit:translate:17.0.3") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-play-services:1.7.3") + + } diff --git a/app/src/main/assets/tessdata/rus.traineddata b/app/src/main/assets/tessdata/rus.traineddata new file mode 100644 index 0000000..702e3c4 Binary files /dev/null and b/app/src/main/assets/tessdata/rus.traineddata differ diff --git a/app/src/main/java/com/akslabs/circletosearch/CircleToSearchAccessibilityService.kt b/app/src/main/java/com/akslabs/circletosearch/CircleToSearchAccessibilityService.kt index fdbc71e..f738693 100644 --- a/app/src/main/java/com/akslabs/circletosearch/CircleToSearchAccessibilityService.kt +++ b/app/src/main/java/com/akslabs/circletosearch/CircleToSearchAccessibilityService.kt @@ -64,6 +64,12 @@ import com.akslabs.circletosearch.ui.components.CopyTextOverlayManager import com.akslabs.circletosearch.utils.ImageUtils import java.util.concurrent.Executor import java.util.concurrent.Executors +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.SupervisorJob +import java.util.concurrent.atomic.AtomicBoolean class CircleToSearchAccessibilityService : AccessibilityService() { @@ -72,6 +78,11 @@ class CircleToSearchAccessibilityService : AccessibilityService() { private val executor: Executor = Executors.newSingleThreadExecutor() private lateinit var configManager: OverlayConfigurationManager + private val serviceJob = SupervisorJob() + private val serviceScope = CoroutineScope(Dispatchers.Default + serviceJob) + + private val isTranslatingFlag = AtomicBoolean(false) + /** Kept by companion so scroll events can re-scan copy-text nodes. */ internal var copyTextManager: CopyTextOverlayManager? = null @@ -104,7 +115,6 @@ class CircleToSearchAccessibilityService : AccessibilityService() { val info = serviceInfo info.flags = info.flags or android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS or - android.accessibilityservice.AccessibilityServiceInfo.FLAG_REQUEST_ENHANCED_WEB_ACCESSIBILITY or android.accessibilityservice.AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS serviceInfo = info @@ -478,7 +488,13 @@ class CircleToSearchAccessibilityService : AccessibilityService() { if (action == ActionType.NONE) return // Haptic feedback for action trigger - val vibrator = getSystemService(Context.VIBRATOR_SERVICE) as Vibrator + val vibrator = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val vibratorManager = getSystemService(Context.VIBRATOR_MANAGER_SERVICE) as android.os.VibratorManager + vibratorManager.defaultVibrator + } else { + @Suppress("DEPRECATION") + getSystemService(Context.VIBRATOR_SERVICE) as Vibrator + } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { vibrator.vibrate(VibrationEffect.createPredefined(VibrationEffect.EFFECT_CLICK)) } else { @@ -684,12 +700,17 @@ class CircleToSearchAccessibilityService : AccessibilityService() { } } - private fun performCapture(searchModeOverride: Boolean? = null) { + private fun performCapture(searchModeOverride: Boolean? = null, translateScreen: Boolean = false) { android.util.Log.d("CircleToSearch", "performCapture called. hasWindowManager=${windowManager != null}") - + // Clear repository at the source to prevent any "ghost" flash of old data BitmapRepository.clear() - + + if (translateScreen && !isTranslatingFlag.compareAndSet(false, true)) { + android.util.Log.d("CircleToSearch", "Translation already in progress, skipping") + return + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { takeScreenshot( Display.DEFAULT_DISPLAY, @@ -699,34 +720,64 @@ class CircleToSearchAccessibilityService : AccessibilityService() { try { val hardwareBuffer = screenshot.hardwareBuffer val colorSpace = screenshot.colorSpace - + val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, colorSpace) + hardwareBuffer.close() // Close buffer after getting bitmap + if (bitmap == null) { - hardwareBuffer.close() + isTranslatingFlag.set(false) return } // Copy to software bitmap val copy = bitmap.copy(Bitmap.Config.ARGB_8888, false) - hardwareBuffer.close() // Close buffer after copy + bitmap.recycle() // Release original bitmap if (copy == null) { + isTranslatingFlag.set(false) return } - - // Store in Repository (In-Memory) - BitmapRepository.setScreenshot(copy) - - // Launch Overlay Immediately - launchOverlay(searchModeOverride) - + + if (translateScreen) { + serviceScope.launch { + try { + val translatedBitmap = ScreenTranslator().use { translator -> + translator.translateScreen(copy) + } + // translateScreen creates its own copy — original copy can be released immediately + copy.recycle() + withContext(Dispatchers.Main) { + BitmapRepository.setScreenshot(translatedBitmap) + launchOverlay(searchModeOverride) + isTranslatingFlag.set(false) + } + } catch (e: Exception) { + android.util.Log.e("CircleToSearch", "Translation pipeline failed", e) + // Translation failed — use original copy + withContext(Dispatchers.Main) { + BitmapRepository.setScreenshot(copy) + launchOverlay(searchModeOverride) + isTranslatingFlag.set(false) + } + } + } + } else { + // Store in Repository (In-Memory) + BitmapRepository.setScreenshot(copy) + + // Launch Overlay Immediately + launchOverlay(searchModeOverride) + } + } catch (e: Exception) { e.printStackTrace() + isTranslatingFlag.set(false) } } override fun onFailure(errorCode: Int) { android.util.Log.e("CircleToSearch", "Screenshot failed with error code: $errorCode") + isTranslatingFlag.set(false) } } ) @@ -750,28 +801,50 @@ class CircleToSearchAccessibilityService : AccessibilityService() { private val rect = RectF() private val matrix = Matrix() private val radius = 12f * context.resources.displayMetrics.density + private var cachedBitmap: Bitmap? = null + private var cachedShader: BitmapShader? = null + + override fun setImageBitmap(bm: Bitmap?) { + cachedBitmap?.recycle() + cachedBitmap = null + cachedShader = null + super.setImageBitmap(bm) + } override fun onDraw(canvas: android.graphics.Canvas) { val drawable = drawable ?: return - val bitmap = try { - drawable.toBitmap() - } catch (e: Exception) { - return + + if (cachedBitmap == null || cachedShader == null) { + cachedBitmap?.recycle() + cachedBitmap = try { + drawable.toBitmap() + } catch (e: Exception) { + return + } + + val shader = BitmapShader(cachedBitmap!!, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP) + cachedShader = shader } - - val shader = BitmapShader(bitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP) - + + val bitmap = cachedBitmap!! + // Adjust shader to current view bounds matrix.reset() matrix.setScale(width.toFloat() / bitmap.width, height.toFloat() / bitmap.height) - shader.setLocalMatrix(matrix) - - paint.shader = shader + cachedShader!!.setLocalMatrix(matrix) + + paint.shader = cachedShader rect.set(0f, 0f, width.toFloat(), height.toFloat()) - + // This never "looses roundness" because it's rendering at the pixel level on each frame canvas.drawRoundRect(rect, radius, radius, paint) } + + fun releaseResources() { + cachedBitmap?.recycle() + cachedBitmap = null + cachedShader = null + } } private fun showPinnedArea(bitmap: Bitmap, rect: android.graphics.Rect) { @@ -1176,6 +1249,11 @@ class CircleToSearchAccessibilityService : AccessibilityService() { android.util.Log.d("CircleToSearch", "triggerCapture static called. instance=${instance != null}") instance?.performCapture(null) } + + fun triggerTranslateCapture() { + android.util.Log.d("CircleToSearch", "triggerTranslateCapture static called. instance=${instance != null}") + instance?.performCapture(null, translateScreen = true) + } fun pinArea(bitmap: Bitmap, rect: android.graphics.Rect) { android.util.Log.d("CircleToSearch", "pinArea static called. instance=${instance != null}") @@ -1195,6 +1273,7 @@ class CircleToSearchAccessibilityService : AccessibilityService() { instance = null prefs.unregisterOnSharedPreferenceChangeListener(prefsListener) overlayPrefs.unregisterOnSharedPreferenceChangeListener(overlayPrefsListener) + serviceJob.cancel() overlayViews.forEach { view -> try { @@ -1206,4 +1285,3 @@ class CircleToSearchAccessibilityService : AccessibilityService() { hideBubble() } } - diff --git a/app/src/main/java/com/akslabs/circletosearch/OverlayActivity.kt b/app/src/main/java/com/akslabs/circletosearch/OverlayActivity.kt index afd5d02..e1a631a 100644 --- a/app/src/main/java/com/akslabs/circletosearch/OverlayActivity.kt +++ b/app/src/main/java/com/akslabs/circletosearch/OverlayActivity.kt @@ -31,14 +31,37 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.unit.dp import com.akslabs.circletosearch.data.BitmapRepository import com.akslabs.circletosearch.ui.CircleToSearchScreen +import com.akslabs.circletosearch.utils.UIPreferences import com.akslabs.circletosearch.ui.components.CopyTextOverlayManager import com.akslabs.circletosearch.ui.theme.CircleToSearchTheme +import androidx.lifecycle.lifecycleScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import android.widget.Toast +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.clickable +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.runtime.remember +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Translate +import androidx.compose.material3.ExtendedFloatingActionButton +import androidx.compose.material3.Icon +import androidx.compose.material3.Text +import androidx.compose.ui.Alignment class OverlayActivity : ComponentActivity() { - - private val screenshotBitmap = androidx.compose.runtime.mutableStateOf(null) + private val copyTextManager = androidx.compose.runtime.mutableStateOf(null) private val searchModeOverride = androidx.compose.runtime.mutableStateOf(null) + private val isTranslating = androidx.compose.runtime.mutableStateOf(false) + private val screenshotBitmap = androidx.compose.runtime.mutableStateOf(null) override fun onCreate(savedInstanceState: Bundle?) { window.setBackgroundDrawable(android.graphics.drawable.ColorDrawable(0)) @@ -67,19 +90,41 @@ class OverlayActivity : ComponentActivity() { color = Color.Transparent, tonalElevation = 0.dp ) { - CircleToSearchScreen( - screenshot = screenshotBitmap.value, - searchModeOverride = searchModeOverride.value, - onClose = { - BitmapRepository.clear() - com.akslabs.circletosearch.data.AssistDataRepository.clear() - finish() - }, - copyTextManager = copyTextManager.value, - onExitCopyMode = { - // Copy Mode exited + Box(modifier = Modifier.fillMaxSize()) { + CircleToSearchScreen( + screenshot = screenshotBitmap.value, + searchModeOverride = searchModeOverride.value, + onClose = { + BitmapRepository.clear() + com.akslabs.circletosearch.data.AssistDataRepository.clear() + finish() + }, + copyTextManager = copyTextManager.value, + onExitCopyMode = { + // Copy Mode exited + }, + onTranslate = { + val targetLang = UIPreferences(this@OverlayActivity).getTargetTranslateLang() + translateCurrentScreen(targetLang) + } + ) + + if (isTranslating.value) { + Box( + modifier = Modifier + .fillMaxSize() + .background(Color.Black.copy(alpha = 0.6f)) + .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) {}, + contentAlignment = Alignment.Center + ) { + Column(horizontalAlignment = Alignment.CenterHorizontally) { + CircularProgressIndicator(color = Color.White) + Spacer(modifier = Modifier.height(16.dp)) + Text("Translating screen...", color = Color.White) + } + } } - ) + } } } } @@ -99,7 +144,7 @@ class OverlayActivity : ComponentActivity() { loadScreenshot() updateOverride(intent) - + // Recreate manager with new screenshot copyTextManager.value = CopyTextOverlayManager( context = this, @@ -116,6 +161,43 @@ class OverlayActivity : ComponentActivity() { } } + fun translateCurrentScreen(targetLangCode: String? = null) { + val currentBitmap = screenshotBitmap.value ?: return + + if (currentBitmap.isRecycled) { + Toast.makeText(this, "Image is no longer available", Toast.LENGTH_SHORT).show() + return + } + + if (isTranslating.value) return + isTranslating.value = true + + lifecycleScope.launch(Dispatchers.Default) { + try { + val translatedBitmap = ScreenTranslator().use { translator -> + translator.translateScreen(currentBitmap, targetLangCode) + } + withContext(Dispatchers.Main) { + val oldBitmap = screenshotBitmap.value + screenshotBitmap.value = translatedBitmap + BitmapRepository.setScreenshot(translatedBitmap) + isTranslating.value = false + + // Recycle old bitmap (original screenshot) after translation + if (oldBitmap != null && !oldBitmap.isRecycled && oldBitmap != translatedBitmap) { + oldBitmap.recycle() + } + } + } catch (e: Exception) { + android.util.Log.e("OverlayActivity", "Translation failed", e) + withContext(Dispatchers.Main) { + Toast.makeText(this@OverlayActivity, e.message ?: "Translation failed", Toast.LENGTH_LONG).show() + isTranslating.value = false + } + } + } + } + private fun loadScreenshot() { val bitmap = BitmapRepository.getScreenshot() if (bitmap != null) { @@ -130,8 +212,11 @@ class OverlayActivity : ComponentActivity() { super.onDestroy() copyTextManager.value?.dismiss() copyTextManager.value = null + + // Compose unbinds it naturally, GC handles it + screenshotBitmap.value = null + if (isFinishing) { - com.akslabs.circletosearch.data.BitmapRepository.clear() com.akslabs.circletosearch.data.AssistDataRepository.clear() com.akslabs.circletosearch.utils.StorageUtils.clearAppCache(this) } diff --git a/app/src/main/java/com/akslabs/circletosearch/ScreenTranslator.kt b/app/src/main/java/com/akslabs/circletosearch/ScreenTranslator.kt new file mode 100644 index 0000000..e23ea4d --- /dev/null +++ b/app/src/main/java/com/akslabs/circletosearch/ScreenTranslator.kt @@ -0,0 +1,261 @@ +package com.akslabs.circletosearch + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Rect +import android.text.Layout +import android.text.StaticLayout +import android.text.TextPaint +import com.google.mlkit.nl.languageid.LanguageIdentification +import com.google.mlkit.nl.translate.TranslateLanguage +import com.google.mlkit.nl.translate.Translation +import com.google.mlkit.nl.translate.Translator +import com.google.mlkit.nl.translate.TranslatorOptions +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions +import kotlinx.coroutines.tasks.await +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.Closeable + +private const val TAG = "ScreenTranslator" + +data class TextBlockData(val text: String, val boundingBox: Rect) +data class TranslatedBlockData(val translatedText: String, val boundingBox: Rect) + +class ScreenTranslator : Closeable { + + private val textRecognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + private val languageIdentifier = LanguageIdentification.getClient() + + private val translators = mutableMapOf() + + suspend fun translateScreen(screenshot: Bitmap, targetLangCode: String? = null): Bitmap { + return withContext(Dispatchers.Default) { + if (screenshot.isRecycled) { + throw IllegalStateException("Bitmap is already recycled.") + } + + val resultBitmap = try { + screenshot.copy(Bitmap.Config.ARGB_8888, true) + } catch (e: OutOfMemoryError) { + throw IllegalStateException("Not enough memory to process screenshot", e) + } + + if (resultBitmap == null) { + throw IllegalStateException("Failed to create bitmap copy") + } + + val canvas = Canvas(resultBitmap) + + val backgroundPaint = Paint().apply { style = Paint.Style.FILL } + val textPaint = TextPaint().apply { + isAntiAlias = true + } + + val textBlocks = recognizeTextWithBounds(screenshot) + + val translatedBlocks = translateBlocks(textBlocks, targetLangCode) + + for (block in translatedBlocks) { + val dominantBgColor = getDominantEdgeColor(screenshot, block.boundingBox) + backgroundPaint.color = dominantBgColor + + val bgRect = Rect(block.boundingBox).apply { inset(-2, -2) } + canvas.drawRect(bgRect, backgroundPaint) + + textPaint.color = getContrastColor(dominantBgColor) + drawMultilineTextToFit(canvas, block.translatedText, block.boundingBox, textPaint) + } + resultBitmap + } + } + + private suspend fun recognizeTextWithBounds(bitmap: Bitmap): List { + val image = InputImage.fromBitmap(bitmap, 0) + val visionText = textRecognizer.process(image).await() + + val resultList = mutableListOf() + for (block in visionText.textBlocks) { + val boundingBox = block.boundingBox ?: continue + val originalText = block.text + + if (originalText.isNotBlank()) { + resultList.add(TextBlockData(originalText, boundingBox)) + } + } + return resultList + } + + private fun getTranslator(sourceLang: String, targetLang: String): Translator { + val key = "$sourceLang-$targetLang" + return translators.getOrPut(key) { + val options = TranslatorOptions.Builder() + .setSourceLanguage(sourceLang) + .setTargetLanguage(targetLang) + .build() + Translation.getClient(options) + } + } + + /** + * Translates recognized text blocks. + * Edge cases: unidentified language, missing model, network errors. + */ + private suspend fun translateBlocks(blocks: List, targetLangCode: String?): List { + val translatedList = mutableListOf() + val langToUse = targetLangCode ?: java.util.Locale.getDefault().language + val targetLanguage = TranslateLanguage.fromLanguageTag(langToUse) ?: TranslateLanguage.ENGLISH + + for (block in blocks) { + try { + val langCode = languageIdentifier.identifyLanguage(block.text).await() + + // Edge case: language not identified ("und") — skip block + if (langCode == "und" || langCode.isNullOrEmpty()) { + continue + } + + val sourceLang = TranslateLanguage.fromLanguageTag(langCode) + + // Edge case: language not supported by ML Kit — skip + if (sourceLang == null || sourceLang == targetLanguage) { + continue + } + + val translator = getTranslator(sourceLang, targetLanguage) + + // Edge case: GrapheneOS / offline mode — model not downloaded + try { + translator.downloadModelIfNeeded().await() + } catch (e: Exception) { + android.util.Log.w(TAG, "Translation model unavailable: $sourceLang -> $targetLanguage. Skipping block.") + continue + } + + val translatedText = translator.translate(block.text).await() + + // Edge case: empty translation or translation matches original + if (translatedText.isNotBlank() && translatedText != block.text) { + translatedList.add(TranslatedBlockData(translatedText, block.boundingBox)) + } + } catch (e: Exception) { + android.util.Log.w(TAG, "Error translating block: '${block.text.take(50)}...'", e) + } + } + return translatedList + } + + /** + * Color Sampler: iterates around BoundingBox perimeter to find dominant background color. + * Edge cases: empty bounds, zero area, bounds outside bitmap. + */ + private fun getDominantEdgeColor(bitmap: Bitmap, bounds: Rect): Int { + val colorCounts = mutableMapOf() + + // OPTIMIZATION: Expand the box outward (padding) to move from the font to the clean background. + val padding = 4 + val left = (bounds.left - padding).coerceIn(0, bitmap.width - 1) + val right = (bounds.right + padding).coerceIn(0, bitmap.width - 1) + val top = (bounds.top - padding).coerceIn(0, bitmap.height - 1) + val bottom = (bounds.bottom + padding).coerceIn(0, bitmap.height - 1) + + // Edge case: empty or zero rect + if (right <= left || bottom <= top) { + return Color.WHITE + } + + // Edge case: very narrow/short boundary — pick single point + if (right == left && bottom == top) { + return bitmap.getPixel(left, top) + } + + val xStep = maxOf(2, (right - left) / 50) + for (x in left..right step xStep) { + val colorTop = bitmap.getPixel(x, top) + val colorBottom = bitmap.getPixel(x, bottom) + colorCounts[colorTop] = (colorCounts[colorTop] ?: 0) + 1 + colorCounts[colorBottom] = (colorCounts[colorBottom] ?: 0) + 1 + } + + val yStep = maxOf(2, (bottom - top) / 50) + for (y in top..bottom step yStep) { + val colorLeft = bitmap.getPixel(left, y) + val colorRight = bitmap.getPixel(right, y) + colorCounts[colorLeft] = (colorCounts[colorLeft] ?: 0) + 1 + colorCounts[colorRight] = (colorCounts[colorRight] ?: 0) + 1 + } + + return colorCounts.maxByOrNull { it.value }?.key ?: Color.WHITE + } + + private fun drawMultilineTextToFit(canvas: Canvas, text: String, rect: Rect, paint: TextPaint) { + var minSize = 10f + var maxSize = 120f + var bestSize = minSize + var bestLayout: StaticLayout? = null + + val textWidth = rect.width().coerceAtLeast(1) + + while (minSize <= maxSize) { + val midSize = (minSize + maxSize) / 2 + paint.textSize = midSize + + val layout = StaticLayout.Builder.obtain(text, 0, text.length, paint, textWidth) + .setAlignment(Layout.Alignment.ALIGN_CENTER) + .setLineSpacing(0f, 1f) + .setIncludePad(false) + .build() + + if (layout.height <= rect.height()) { + bestSize = midSize + bestLayout = layout + minSize = midSize + 1f + } else { + maxSize = midSize - 1f + } + } + + if (bestLayout != null) { + canvas.save() + val textY = rect.centerY() - (bestLayout.height / 2f) + canvas.translate(rect.left.toFloat(), textY) + bestLayout.draw(canvas) + canvas.restore() + } + } + + /** + * WCAG 2.1 relative luminance + contrast ratio. + * Returns BLACK or WHITE for guaranteed readable text. + */ + private fun getContrastColor(backgroundColor: Int): Int { + // WCAG relative luminance + fun channelLuminance(c: Int): Double { + val sRGB = c / 255.0 + return if (sRGB <= 0.03928) sRGB / 12.92 else Math.pow((sRGB + 0.055) / 1.055, 2.4) + } + val r = Color.red(backgroundColor) + val g = Color.green(backgroundColor) + val b = Color.blue(backgroundColor) + val luminance = 0.2126 * channelLuminance(r) + + 0.7152 * channelLuminance(g) + + 0.0722 * channelLuminance(b) + + // WCAG Formula: (L1 + 0.05) / (L2 + 0.05) where L1 is the lighter color + val contrastWithWhite = 1.05 / (luminance + 0.05) + val contrastWithBlack = (luminance + 0.05) / 0.05 + return if (contrastWithBlack > contrastWithWhite) Color.BLACK else Color.WHITE + } + + override fun close() { + textRecognizer.close() + languageIdentifier.close() + translators.values.forEach { it.close() } + translators.clear() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/akslabs/circletosearch/ocr/TesseractEngine.kt b/app/src/main/java/com/akslabs/circletosearch/ocr/TesseractEngine.kt index f079ce7..5938c7b 100644 --- a/app/src/main/java/com/akslabs/circletosearch/ocr/TesseractEngine.kt +++ b/app/src/main/java/com/akslabs/circletosearch/ocr/TesseractEngine.kt @@ -13,6 +13,8 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import java.io.File import java.io.FileOutputStream import java.util.UUID @@ -21,24 +23,99 @@ object TesseractEngine { private const val TAG = "TesseractEngine" private var isPrepared = false + // Cache Tesseract instance to avoid loading 30MB+ dictionaries from disk on every scan + private var cachedTessApi: TessBaseAPI? = null + private var cachedLang: String? = null + private val ocrMutex = Mutex() + + // Default languages: eng + rus for Cyrillic support + private val defaultLanguages = listOf("eng", "rus") + + private val latinToCyrillic = mapOf( + 'A' to 'А', 'a' to 'а', + 'B' to 'В', + 'C' to 'С', 'c' to 'с', + 'E' to 'Е', 'e' to 'е', + 'H' to 'Н', + 'K' to 'К', 'k' to 'к', + 'M' to 'М', 'm' to 'м', + 'O' to 'О', 'o' to 'о', + 'P' to 'Р', 'p' to 'р', + 'T' to 'Т', 't' to 'т', + 'X' to 'Х', 'x' to 'х', + 'Y' to 'У', 'y' to 'у' + ) + private val cyrillicToLatin = latinToCyrillic.entries.associate { (k, v) -> v to k } + + private fun cleanWordText(text: String, lineDominantCyrillic: Boolean): String { + // Remove frequent OCR artifacts (including noise like "<") + var t = text.replace("<", "").replace(">", "").trim() + if (t.isEmpty()) return "" + + val cCyr = t.count { it in 'А'..'я' || it == 'Ё' || it == 'ё' } + val cLat = t.count { it in 'A'..'Z' || it in 'a'..'z' } + + if (cCyr == 0 && cLat == 0) return t + + if (cCyr > 0 && cLat > 0) { + val toCyrillic = cCyr >= cLat + t = t.map { char -> + if (toCyrillic && latinToCyrillic.containsKey(char)) latinToCyrillic[char]!! + else if (!toCyrillic && cyrillicToLatin.containsKey(char)) cyrillicToLatin[char]!! + else char + }.joinToString("") + } else if (lineDominantCyrillic && cLat > 0 && cCyr == 0) { + val onlyLookalikes = t.all { !it.isLetter() || latinToCyrillic.containsKey(it) } + if (onlyLookalikes) t = t.map { char -> latinToCyrillic[char] ?: char }.joinToString("") + } else if (!lineDominantCyrillic && cCyr > 0 && cLat == 0) { + val onlyLookalikes = t.all { !it.isLetter() || cyrillicToLatin.containsKey(it) } + if (onlyLookalikes) t = t.map { char -> cyrillicToLatin[char] ?: char }.joinToString("") + } + + return t + } + + private fun calculateAverageLuminance(bitmap: Bitmap): Float { + val stepX = maxOf(1, bitmap.width / 50) + val stepY = maxOf(1, bitmap.height / 50) + var sumLuminance = 0f + var count = 0 + for (x in 0 until bitmap.width step stepX) { + for (y in 0 until bitmap.height step stepY) { + val pixel = bitmap.getPixel(x, y) + val r = android.graphics.Color.red(pixel) + val g = android.graphics.Color.green(pixel) + val b = android.graphics.Color.blue(pixel) + sumLuminance += (0.299f * r + 0.587f * g + 0.114f * b) + count++ + } + } + return if (count > 0) sumLuminance / count else 255f + } + fun prepareTessData(context: Context): String { val filesDir = context.filesDir.absolutePath + // OPTIMIZATION: Immediate exit if files are already verified in this session + if (isPrepared) return filesDir val tessDir = File(filesDir, "tessdata") if (!tessDir.exists()) { tessDir.mkdirs() } - val engFile = File(tessDir, "eng.traineddata") - if (!engFile.exists()) { - Log.d(TAG, "Copying eng.traineddata from assets...") - try { - context.assets.open("tessdata/eng.traineddata").use { input -> - FileOutputStream(engFile).use { output -> - input.copyTo(output) + // Copy default models + for (lang in defaultLanguages) { + val langFile = File(tessDir, "$lang.traineddata") + if (!langFile.exists()) { + Log.d(TAG, "Copying $lang.traineddata from assets...") + try { + context.assets.open("tessdata/$lang.traineddata").use { input -> + FileOutputStream(langFile).use { output -> + input.copyTo(output) + } } + } catch (e: Exception) { + Log.e(TAG, "Failed to copy $lang.traineddata: ${e.message}") } - } catch (e: Exception) { - Log.e(TAG, "Failed to copy eng.traineddata: ${e.message}") } } isPrepared = true @@ -50,11 +127,57 @@ object TesseractEngine { if (!dir.exists()) { prepareTessData(context) } - + val files = dir.listFiles() ?: return listOf("eng") - return files.filter { it.name.endsWith(".traineddata") } + val available = files.filter { it.name.endsWith(".traineddata") } .map { it.name.removeSuffix(".traineddata") } .sorted() + + // Fallback to eng if no models found + return available.ifEmpty { listOf("eng") } + } + + /** + * Automatically detects language based on system settings. + * Returns preferred language for OCR. + */ + fun getSystemLanguage(): String { + // Priority is given to Russian ("rus+eng") to prevent + // Cyrillic words from being converted into lookalike English ones (e.g., "тому" -> "tommy"). + return "rus+eng" + } + + /** + * Returns language for OCR - either system default or user saved. + */ + fun getOcrLanguage(context: Context): String { + val prefs = context.getSharedPreferences("OcrSettings", Context.MODE_PRIVATE) + return prefs.getString("selected_lang", "rus+eng") ?: "rus+eng" + } + + /** + * Checks if model is available for specified language. + */ + fun isModelAvailable(context: Context, lang: String): Boolean { + val tessDir = File(context.filesDir, "tessdata") + // Support combined languages like "rus+eng" + return lang.split("+").all { File(tessDir, "$it.traineddata").exists() } + } + + /** + * Checks for Russian model and offers download if missing. + */ + fun checkAndOfferRussianModel(context: Context): Boolean { + if (isModelAvailable(context, "rus")) return true + + val prefs = context.getSharedPreferences("OcrSettings", Context.MODE_PRIVATE) + val offerShown = prefs.getBoolean("rus_model_offer_shown", false) + + if (!offerShown) { + prefs.edit().putBoolean("rus_model_offer_shown", true).apply() + return false + } + return false } fun importModel(context: Context, uri: android.net.Uri, callback: (Boolean, String) -> Unit) { @@ -95,64 +218,83 @@ object TesseractEngine { } /** - * Tiled extraction: Runs multiple OCR passes in parallel and merges results. - * This mirrors the QR scanner's multi-resolution logic to maximize accuracy. + * Extracts text from a bitmap using a cached Tesseract instance for massive speed improvements. */ - suspend fun extractText(context: Context, bitmap: Bitmap): List = coroutineScope { - val dataPath = withContext(Dispatchers.IO) { prepareTessData(context) } - val prefs = context.getSharedPreferences("OcrSettings", Context.MODE_PRIVATE) - val lang = prefs.getString("selected_lang", "eng") ?: "eng" - - val w = bitmap.width - val h = bitmap.height - - // Define Tiles (Full + 2x2 Grid with 20% overlap) - val tiles = mutableListOf() - // 1. Full - tiles.add(Rect(0, 0, w, h)) - - // 2. 2x2 Grid (Each tile is ~60% size to provide nice overlap) - val tw = (w * 0.6f).toInt() - val th = (h * 0.6f).toInt() - tiles.add(Rect(0, 0, tw, th)) // Top Left - tiles.add(Rect(w - tw, 0, w, th)) // Top Right - tiles.add(Rect(0, h - th, tw, h)) // Bottom Left - tiles.add(Rect(w - tw, h - th, w, h)) // Bottom Right - - val allPasses = tiles.mapIndexed { index, rect -> - async(Dispatchers.Default) { - index to internalExtractWords(dataPath, lang, bitmap, rect) + suspend fun extractText(context: Context, bitmap: Bitmap): List = withContext(Dispatchers.IO) { + val dataPath = prepareTessData(context) + // Use automatic language detection + val lang = getOcrLanguage(context) + + val words = ocrMutex.withLock { + if (cachedTessApi == null || cachedLang != lang) { + cachedTessApi?.recycle() + cachedTessApi = TessBaseAPI() + val success = cachedTessApi?.init(dataPath, lang) ?: false + if (!success) { + cachedTessApi?.recycle() + cachedTessApi = null + return@withLock emptyList() + } + cachedLang = lang } - } - val allWordsWithSource = allPasses.awaitAll().flatMap { (index, words) -> - words.map { index to it } - } - - // Final Merge & Line Grouping - groupWordsIntoNodes(allWordsWithSource) - } + // SPEED AND QUALITY OPTIMIZATION: + // Scaling + Conversion to B&W with high contrast. + val maxDim = Math.max(bitmap.width, bitmap.height).toFloat() + val targetMax = 2048f + // UI text can be small. Scale by 1.5x up to a hard cap of ~2048px. + val scaleFactor = Math.min(1.5f, targetMax / maxDim).coerceAtLeast(1f) + + val scaledWidth = (bitmap.width * scaleFactor).toInt() + val scaledHeight = (bitmap.height * scaleFactor).toInt() + + val processedBitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888) + val canvas = android.graphics.Canvas(processedBitmap) + val paint = android.graphics.Paint(android.graphics.Paint.FILTER_BITMAP_FLAG) // Bilinear filtering + + // Smart Binarization & Inversion + val avgLuminance = calculateAverageLuminance(bitmap) + val isDarkMode = avgLuminance < 128f - private fun internalExtractWords(dataPath: String, lang: String, fullBitmap: Bitmap, crop: Rect): List { - val words = mutableListOf() - val tess = TessBaseAPI() - try { - if (!tess.init(dataPath, lang)) return emptyList() + val colorMatrix = android.graphics.ColorMatrix() + colorMatrix.setSaturation(0f) // Grayscale - // If the crop is a sub-region, create a subset bitmap - val tileBitmap = if (crop.left == 0 && crop.top == 0 && crop.width() == fullBitmap.width && crop.height() == fullBitmap.height) { - fullBitmap - } else { - Bitmap.createBitmap(fullBitmap, crop.left, crop.top, crop.width(), crop.height()) + if (isDarkMode) { + // Invert the image (makes text black and backgrounds white) + colorMatrix.postConcat(android.graphics.ColorMatrix(floatArrayOf( + -1f, 0f, 0f, 0f, 255f, + 0f, -1f, 0f, 0f, 255f, + 0f, 0f, -1f, 0f, 255f, + 0f, 0f, 0f, 1f, 0f + ))) } - tess.setImage(tileBitmap) - tess.getUTF8Text() // Trigger recognition + // Harsh contrast multiplier to force binarization and crush background noise + val contrast = 5.0f + val translate = (-128f * contrast) + 128f + colorMatrix.postConcat(android.graphics.ColorMatrix(floatArrayOf( + contrast, 0f, 0f, 0f, translate, + 0f, contrast, 0f, 0f, translate, + 0f, 0f, contrast, 0f, translate, + 0f, 0f, 0f, 1f, 0f + ))) + + paint.colorFilter = android.graphics.ColorMatrixColorFilter(colorMatrix) + + val matrix = android.graphics.Matrix() + matrix.postScale(scaleFactor, scaleFactor) + canvas.drawBitmap(bitmap, matrix, paint) - val iterator = tess.resultIterator ?: run { - tess.recycle() - return emptyList() - } + val api = cachedTessApi!! + + // Set Page Segmentation Mode to Sparse Text (11) to handle non-linear chat bubbles + api.pageSegMode = TessBaseAPI.PageSegMode.PSM_SPARSE_TEXT + + api.setImage(processedBitmap) + api.getUTF8Text() // Trigger recognition + + val iterator = api.resultIterator ?: return@withLock emptyList() + val extractedWords = mutableListOf() iterator.begin() do { @@ -172,33 +314,34 @@ object TesseractEngine { if (wRect.isEmpty || wRect.width() < 2) continue - // Adjust coordinates to global screen space - val globalBounds = RectF( - (wRect.left + crop.left).toFloat(), - (wRect.top + crop.top).toFloat(), - (wRect.right + crop.left).toFloat(), - (wRect.bottom + crop.top).toFloat() - ) - - words.add( + extractedWords.add( Word( text = wordText, - index = 0, // Assigned later + index = 0, startIndex = 0, endIndex = wordText.length, - bounds = globalBounds + // Return coordinates back to the original screen scale + bounds = RectF( + wRect.left / scaleFactor, + wRect.top / scaleFactor, + wRect.right / scaleFactor, + wRect.bottom / scaleFactor + ) ) ) - } while (iterator.next(TessBaseAPI.PageIteratorLevel.RIL_WORD)) iterator.delete() - } catch (e: Exception) { - Log.e(TAG, "Tile process error: ${e.message}") - } finally { - tess.recycle() + api.clear() // Clear image buffer from native memory to prevent leaks, but keep models loaded + processedBitmap.recycle() + extractedWords } - return words + + // Wrapper to maintain compatibility with groupWordsIntoNodes + val allWordsWithSource = words.map { 0 to it } + + // Final Merge & Line Grouping + groupWordsIntoNodes(allWordsWithSource) } private fun groupWordsIntoNodes(allWordsWithSource: List>): List { @@ -208,7 +351,7 @@ object TesseractEngine { // We prefer results from quadrants (indices 1-4) over the full pass (index 0) // because zoomed-in crops generally yield higher accuracy for small text. val uniqueWords = mutableListOf() - val sortedByPreference = allWordsWithSource.sortedWith(compareByDescending> { it.first }.thenByDescending { it.second.bounds.width() * it.second.bounds.height() }) + val sortedByPreference = allWordsWithSource.sortedWith(compareByDescending> { it.first }.thenBy { it.second.bounds.width() * it.second.bounds.height() }) for (pair in sortedByPreference) { val w = pair.second @@ -217,63 +360,83 @@ object TesseractEngine { if (overlap.intersect(existing.bounds)) { val overlapArea = overlap.width() * overlap.height() val wArea = w.bounds.width() * w.bounds.height() - val textMatch = w.text.equals(existing.text, ignoreCase = true) - // If text matches and there is significant overlap, it's a duplicate - overlapArea > wArea * 0.7 && textMatch + val existingArea = existing.bounds.width() * existing.bounds.height() + // If overlap area is more than 50% of the SMALLEST word, it's a duplicate. + overlapArea > minOf(wArea, existingArea) * 0.5f } else false } if (!isDuplicate) uniqueWords.add(w) } - // 2. Line Clustering (Vertical Overlap) + // 2. Line Clustering (Vertical Overlap & Horizontal Proximity) val sortedWords = uniqueWords.sortedBy { it.bounds.top } val lines = mutableListOf>() - if (sortedWords.isNotEmpty()) { - var currentLine = mutableListOf() - currentLine.add(sortedWords[0]) - lines.add(currentLine) + for (word in sortedWords) { + var addedToLine = false - for (i in 1 until sortedWords.size) { - val prev = currentLine.last() - val curr = sortedWords[i] - - // If vertical centers are close enough, they are on the same line - val verticalOverlap = Math.abs(curr.bounds.centerY() - prev.bounds.centerY()) < (prev.bounds.height() * 0.6f) + for (line in lines) { + val referenceWord = line.first() + val avgHeight = (referenceWord.bounds.height() + word.bounds.height()) / 2f + val verticalOverlap = Math.abs(word.bounds.centerY() - referenceWord.bounds.centerY()) < (avgHeight * 0.6f) if (verticalOverlap) { - currentLine.add(curr) - } else { - currentLine = mutableListOf(curr) - lines.add(currentLine) + // If words are at the same level, check that they are close horizontally + val maxGap = avgHeight * 3.5f + val isCloseHorizontally = line.any { w -> + val gap1 = word.bounds.left - w.bounds.right + val gap2 = w.bounds.left - word.bounds.right + maxOf(gap1, gap2) < maxGap + } + + if (isCloseHorizontally) { + line.add(word) + addedToLine = true + break + } } } + + if (!addedToLine) { + lines.add(mutableListOf(word)) + } } // 3. Horizontal Sorting & Node Construction val result = mutableListOf() lines.forEach { lineWords -> val finalLineWords = lineWords.sortedBy { it.bounds.left } - val fullText = finalLineWords.joinToString(" ") { it.text } + val lineText = finalLineWords.joinToString(" ") { it.text } + val cCyr = lineText.count { it in 'А'..'я' || it == 'Ё' || it == 'ё' } + val cLat = lineText.count { it in 'A'..'Z' || it in 'a'..'z' } + val lineDominantCyrillic = cCyr >= cLat + + val cleanedWords = mutableListOf() val lineBounds = Rect() + finalLineWords.forEachIndexed { idx, w -> - // Re-index words for the line - finalLineWords[idx].copy(index = idx) - - val r = Rect() - w.bounds.roundOut(r) - if (lineBounds.isEmpty()) lineBounds.set(r) else lineBounds.union(r) + val cleanedText = cleanWordText(w.text, lineDominantCyrillic) + if (cleanedText.isNotBlank()) { + val r = Rect() + w.bounds.roundOut(r) + if (lineBounds.isEmpty) lineBounds.set(r) else lineBounds.union(r) + + cleanedWords.add(w.copy(index = cleanedWords.size, text = cleanedText)) + } } - result.add( - TextNode( - id = UUID.randomUUID().toString(), - fullText = fullText, - bounds = lineBounds, - words = finalLineWords + if (cleanedWords.isNotEmpty()) { + val fullText = cleanedWords.joinToString(" ") { it.text } + result.add( + TextNode( + id = UUID.randomUUID().toString(), + fullText = fullText, + bounds = lineBounds, + words = cleanedWords + ) ) - ) + } } return result diff --git a/app/src/main/java/com/akslabs/circletosearch/ui/CircleToSearchScreen.kt b/app/src/main/java/com/akslabs/circletosearch/ui/CircleToSearchScreen.kt index a7795ac..5539531 100644 --- a/app/src/main/java/com/akslabs/circletosearch/ui/CircleToSearchScreen.kt +++ b/app/src/main/java/com/akslabs/circletosearch/ui/CircleToSearchScreen.kt @@ -47,6 +47,7 @@ import androidx.compose.foundation.clickable import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.verticalScroll import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.interaction.MutableInteractionSource import androidx.compose.foundation.interaction.collectIsPressedAsState @@ -84,7 +85,7 @@ import androidx.compose.material.icons.filled.LightMode import androidx.compose.material.icons.filled.Mic import androidx.compose.material.icons.filled.MoreVert import androidx.compose.material.icons.filled.MusicNote -import androidx.compose.material.icons.filled.OpenInNew +import androidx.compose.material.icons.automirrored.filled.OpenInNew import androidx.compose.material.icons.filled.QrCode import androidx.compose.material.icons.filled.PushPin import androidx.compose.material.icons.filled.Refresh @@ -172,6 +173,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.io.ByteArrayOutputStream +import kotlinx.coroutines.tasks.await import androidx.webkit.WebViewFeature import com.akslabs.circletosearch.data.isDirectUpload import kotlin.math.max @@ -191,7 +193,8 @@ fun CircleToSearchScreen( searchModeOverride: Boolean? = null, copyTextManager: com.akslabs.circletosearch.ui.components.CopyTextOverlayManager? = null, onCopyText: () -> Unit = {}, - onExitCopyMode: () -> Unit = {} + onExitCopyMode: () -> Unit = {}, + onTranslate: () -> Unit = {} ) { val context = LocalContext.current val scope = rememberCoroutineScope() @@ -253,16 +256,12 @@ fun CircleToSearchScreen( } val searchEngines = preferredOrder - // Copy Mode internal state - var isCopyMode by remember { mutableStateOf(false) } - // Support Settings Sheet var showSettingsScreen by remember { mutableStateOf(false) } // Friendly Message State var friendlyMessage by remember { mutableStateOf("") } var isMessageVisible by remember { mutableStateOf(false) } - var isCopyTextTriggered by remember { mutableStateOf(false) } // Resizing state var isResizing by remember { mutableStateOf(false) } @@ -276,9 +275,12 @@ fun CircleToSearchScreen( // Phase 44: Smart Entity Extractor var isEntityExtractMode by remember { mutableStateOf(false) } + var isCopyMode by remember { mutableStateOf(false) } var detectedEntities by remember { mutableStateOf>(emptyList()) } var isExtractingEntities by remember { mutableStateOf(false) } + var showTranslationLangDialog by remember { mutableStateOf(false) } + LaunchedEffect(Unit) { if (uiPreferences.isShowFriendlyMessages()) { val manager = FriendlyMessageManager(context) @@ -412,7 +414,6 @@ fun CircleToSearchScreen( // Lifecycle reset: When screenshot changes, reset selection and modes LaunchedEffect(screenshot) { if (screenshot != null) { - isCopyMode = false selectionRect = null selectedBitmap = null isSearching = false @@ -461,16 +462,10 @@ fun CircleToSearchScreen( settings.apply { javaScriptEnabled = true domStorageEnabled = true - databaseEnabled = true allowFileAccess = true allowContentAccess = true - allowFileAccessFromFileURLs = true - allowUniversalAccessFromFileURLs = true mixedContentMode = WebSettings.MIXED_CONTENT_ALWAYS_ALLOW - // Performance & UI - setRenderPriority(WebSettings.RenderPriority.HIGH) - // Caching for Speed cacheMode = WebSettings.LOAD_DEFAULT // Was LOAD_CACHE_ELSE_NETWORK - caused refresh issues @@ -538,11 +533,6 @@ fun CircleToSearchScreen( // Back Handler Logic BackHandler(enabled = true) { - if (isCopyMode) { - isCopyMode = false - onExitCopyMode() - return@BackHandler - } val currentWebView = webViews[selectedEngine] if (currentWebView != null && currentWebView.canGoBack()) { currentWebView.goBack() @@ -869,7 +859,6 @@ fun CircleToSearchScreen( // Close button for Copy Mode (Top Left) // Friendly Message Overlay (Top Center) - if (!isCopyMode) { Box( modifier = Modifier .fillMaxSize() @@ -882,10 +871,9 @@ fun CircleToSearchScreen( visible = isMessageVisible ) } - } // 1. Screenshot Layer - if (screenshot != null && !isCopyMode) { + if (screenshot != null) { Box( modifier = Modifier .fillMaxSize() @@ -940,7 +928,7 @@ fun CircleToSearchScreen( } // 2. Gradient Border Layer (Overlaying screenshot, clipped to rounded corners) - if (showGradientBorder && !isCopyMode) { + if (showGradientBorder) { androidx.compose.animation.AnimatedVisibility( visible = isUIVisible, enter = androidx.compose.animation.fadeIn(animationSpec = tween(700)) @@ -961,12 +949,20 @@ fun CircleToSearchScreen( } // 3. Drawing Canvas (Interactive Layer) - if (!isCopyMode) { Canvas( - modifier = Modifier - .fillMaxSize() - .pointerInput(Unit) { - detectDragGestures( + Canvas( + modifier = Modifier + .fillMaxSize() + .pointerInput(Unit) { + detectTapGestures( + onTap = { + isUIVisible = !isUIVisible + } + ) + } + .pointerInput(Unit) { + detectDragGestures( onDragStart = { offset -> + isUIVisible = false val rect = selectionRect if (rect != null && selectionAnim.value == 1f) { val handleSize = 64f // px for hit testing @@ -1011,7 +1007,11 @@ fun CircleToSearchScreen( currentPathPoints.add(change.position) } }, + onDragCancel = { + isUIVisible = true + }, onDragEnd = { + isUIVisible = true if (isResizing) { isResizing = false activeHandle = null @@ -1162,15 +1162,17 @@ fun CircleToSearchScreen( } } - } - // 4. Header (Top) androidx.compose.animation.AnimatedVisibility( - visible = isUIVisible && !isCopyMode && !isEntityExtractMode, + visible = isUIVisible && !isEntityExtractMode, enter = androidx.compose.animation.slideInVertically( initialOffsetY = { -it }, // Commence au-dessus de l'écran (-100%) animationSpec = tween(500, easing = androidx.compose.animation.core.FastOutSlowInEasing) ), + exit = androidx.compose.animation.slideOutVertically( + targetOffsetY = { -it }, + animationSpec = tween(300) + ), modifier = Modifier.align(Alignment.TopCenter).zIndex(2000f) ) { Row( @@ -1313,7 +1315,7 @@ fun CircleToSearchScreen( androidx.compose.material3.DropdownMenuItem( text = { Text("Open in Browser") }, leadingIcon = { - Icon(Icons.Default.OpenInNew, contentDescription = null) + Icon(Icons.AutoMirrored.Filled.OpenInNew, contentDescription = null) }, onClick = { val currentUrl = webViews[selectedEngine]?.url ?: searchUrl @@ -1336,6 +1338,16 @@ fun CircleToSearchScreen( } ) androidx.compose.material3.DropdownMenuItem( + text = { Text("Translation Target") }, + leadingIcon = { + Icon(Icons.Default.Translate, contentDescription = null) + }, + onClick = { + showTranslationLangDialog = true + showMenu = false + } + ) + androidx.compose.material3.DropdownMenuItem( text = { Text("Settings") }, leadingIcon = { Icon(Icons.Default.Settings, contentDescription = null) @@ -1350,11 +1362,9 @@ fun CircleToSearchScreen( } } // 5. Bottom Bar — Material 3 Expressive two-row card - // State for Copy Text mode - var isCopyTextTriggered by remember { mutableStateOf(false) } androidx.compose.animation.AnimatedVisibility( - visible = isUIVisible && !isCopyMode && !isEntityExtractMode, + visible = isUIVisible && !isEntityExtractMode, enter = slideInVertically( initialOffsetY = { it }, // slides up from below animationSpec = tween(300, easing = androidx.compose.animation.core.CubicBezierEasing(0f, 0f, 0.2f, 1f)) @@ -1457,43 +1467,11 @@ fun CircleToSearchScreen( } } - // Circular Button: Assist Copy (Music Icon) - val assistNodes by com.akslabs.circletosearch.data.AssistDataRepository.assistNodes.collectAsState() - val isAssistDataReady by com.akslabs.circletosearch.data.AssistDataRepository.isDataReady.collectAsState() - - IconButton( - onClick = { - haptic.performHapticFeedback(androidx.compose.ui.hapticfeedback.HapticFeedbackType.LongPress) - - if (isAssistDataReady) { - // Use high-accuracy AssistStructure data + OCR merge - copyTextManager?.setHybridMode(assistNodes) - isCopyMode = true - isCopyTextTriggered = true - } else { - // Data not ready (likely bubble trigger) - android.widget.Toast.makeText(context, "Comming Soon: launch CTS as assistant to try Hybrid text detection.", android.widget.Toast.LENGTH_LONG).show() - } - }, - modifier = Modifier - .size(60.dp) - .background(MaterialTheme.colorScheme.surfaceContainer, CircleShape), - ) { - Icon(Icons.Default.MusicNote, contentDescription = "Assist Copy", tint = if (isAssistDataReady) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurface) - } - // Circular Button: Translate IconButton( onClick = { haptic.performHapticFeedback(androidx.compose.ui.hapticfeedback.HapticFeedbackType.LongPress) - android.widget.Toast.makeText(context, "Coming soon", android.widget.Toast.LENGTH_SHORT).show() - /* - try { - val intent = context.packageManager.getLaunchIntentForPackage("com.google.android.apps.translate")?.apply { addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK) } - if (intent != null) context.startActivity(intent) - else context.startActivity(android.content.Intent(android.content.Intent.ACTION_VIEW, android.net.Uri.parse("https://translate.google.com")).apply { addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK) }) - } catch (e: Exception) {} - */ + onTranslate() }, modifier = Modifier .size(60.dp) @@ -1626,13 +1604,6 @@ fun CircleToSearchScreen( showDonateSheet = true } - // Copy Text - BottomBarButton("Copy Text", { Icon(painterResource(id = com.akslabs.circletosearch.R.drawable.ocr), null, modifier = Modifier.size(20.dp)) }) { - copyTextManager?.setOcrOnlyMode() - isCopyMode = true - isCopyTextTriggered = true - } - // QR Scan BottomBarButton("Scan QR", { Icon(Icons.Default.QrCode, null) }) { qrScanBitmap = selectedBitmap ?: screenshot @@ -1671,12 +1642,11 @@ fun CircleToSearchScreen( ) } - // Copy Text overlay integration (Activity-based) - if (isCopyMode && copyTextManager != null) { + // Seamless Text Selection Overlay integration (Active when in copy mode) + if (copyTextManager != null) { AndroidView( factory = { ctx -> copyTextManager.getOverlayView(onDismiss = { - isCopyMode = false onExitCopyMode() }) }, @@ -1687,7 +1657,7 @@ fun CircleToSearchScreen( } // 4. Selection Actions (Share) — Positioned at the very end for absolute top-layer rendering - if (selectionRect != null && selectionAnim.value == 1f && !isCopyMode) { + if (selectionRect != null && selectionAnim.value == 1f) { val rect = selectionRect!! val density = androidx.compose.ui.platform.LocalDensity.current val leftPx = rect.left.toFloat() @@ -1811,7 +1781,7 @@ fun CircleToSearchScreen( } // --- NEW: QR Overlay Chips (High Layer) --- - if (screenshot != null && !isCopyMode) { + if (screenshot != null) { BoxWithConstraints(modifier = Modifier.fillMaxSize().zIndex(2500f)) { val screenWidth = maxWidth val screenHeight = maxHeight @@ -1886,7 +1856,7 @@ fun CircleToSearchScreen( } // --- Phase 44: Smart Entity Extractor Overlay Chips --- - if (isEntityExtractMode && screenshot != null && !isCopyMode) { + if (isEntityExtractMode && screenshot != null) { BoxWithConstraints(modifier = Modifier.fillMaxSize().zIndex(2600f)) { val screenWidth = maxWidth val screenHeight = maxHeight @@ -2051,6 +2021,57 @@ fun CircleToSearchScreen( onDismissRequest = { showSettingsScreen = false } ) } + + if (showTranslationLangDialog) { + val languages = listOf( + null to "Auto (System Default)", + "ar" to "Arabic", + "zh" to "Chinese", + "nl" to "Dutch", + "en" to "English", + "fr" to "French", + "de" to "German", + "hi" to "Hindi", + "it" to "Italian", + "ja" to "Japanese", + "ko" to "Korean", + "pl" to "Polish", + "pt" to "Portuguese", + "ru" to "Russian", + "es" to "Spanish", + "tr" to "Turkish", + "uk" to "Ukrainian" + ) + androidx.compose.material3.AlertDialog( + onDismissRequest = { showTranslationLangDialog = false }, + title = { Text("Translate to...") }, + text = { + Column(modifier = Modifier.verticalScroll(rememberScrollState())) { + val currentSelection = uiPreferences.getTargetTranslateLang() + languages.forEach { (code, name) -> + val isSelected = currentSelection == code + Text( + text = name, + modifier = Modifier + .fillMaxWidth() + .clickable { + uiPreferences.setTargetTranslateLang(code) + showTranslationLangDialog = false + } + .padding(vertical = 12.dp, horizontal = 16.dp), + color = if (isSelected) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurface, + fontWeight = if (isSelected) FontWeight.Bold else FontWeight.Normal + ) + } + } + }, + confirmButton = { + androidx.compose.material3.TextButton(onClick = { showTranslationLangDialog = false }) { + Text("Close") + } + } + ) + } } } } diff --git a/app/src/main/java/com/akslabs/circletosearch/ui/components/CopyTextOverlayManager.kt b/app/src/main/java/com/akslabs/circletosearch/ui/components/CopyTextOverlayManager.kt index 804698b..98bd1b9 100644 --- a/app/src/main/java/com/akslabs/circletosearch/ui/components/CopyTextOverlayManager.kt +++ b/app/src/main/java/com/akslabs/circletosearch/ui/components/CopyTextOverlayManager.kt @@ -31,11 +31,8 @@ import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.ui.Alignment import com.akslabs.circletosearch.data.BitmapRepository -import com.akslabs.circletosearch.ocr.TesseractEngine import com.akslabs.circletosearch.utils.ImageUtils import kotlinx.coroutines.* -import java.util.UUID - /** Simple holder for a floating-toolbar button's label and screen hit-rect. */ private class ToolbarButton(val label: String, val rect: Rect) @@ -50,43 +47,40 @@ class CopyTextOverlayManager( private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main) private var scanJob: Job? = null private var onDismissCallback: (() -> Unit)? = null + private var onAnalysisCompleteCallback: ((Int) -> Unit)? = null - // State for Compose and Drawing private val isScanning = mutableStateOf(false) - private val textNodes = mutableStateListOf() - private var allWords: List = emptyList() - - // Status message for the user private val statusMessage = mutableStateOf(null) - - // Assistant Data support - var isAssistMode: Boolean = false - private set - - private var nativeNodes: List = emptyList() - - fun setHybridMode(nodes: List) { - isAssistMode = true - nativeNodes = nodes - textNodes.clear() - textNodes.addAll(nodes) - updateAllWords() + private val textNodes = mutableListOf() + private var allWords: List = emptyList() + + /** + * Sets a callback that is invoked after the analysis is complete. + * @param callback receives the number of text nodes found. + */ + fun setOnAnalysisComplete(callback: (Int) -> Unit) { + onAnalysisCompleteCallback = callback } - - fun setOcrOnlyMode() { - isAssistMode = false - nativeNodes = emptyList() - textNodes.clear() - allWords = emptyList() - statusMessage.value = null + + /** + * Starts text analysis. Called automatically on startup. + */ + fun startAnalysis() { + // No-op: getOverlayView already calls scanNodes natively. Prevents double-scanning. } - + + /** + * Returns the number of found text nodes. + */ + fun getNodeCount(): Int = textNodes.size + + /** + * Checks if scanning is currently in progress. + */ + fun isScanning(): Boolean = isScanning.value + private fun updateAllWords() { - allWords = textNodes.flatMap { node -> - node.words.map { word -> - word - } - } + allWords = textNodes.flatMap { it.words } } // Selection state @@ -124,7 +118,7 @@ class CopyTextOverlayManager( ) Spacer(Modifier.height(16.dp)) Text( - if (isAssistMode) "Hybrid Deep Scan..." else "Scanning text...", + "Scanning text...", style = MaterialTheme.typography.titleMedium, color = ComposeColor.White, modifier = Modifier @@ -155,26 +149,13 @@ class CopyTextOverlayManager( } container.addView(topBar) - if (isAssistMode) { - // Trigger parallel OCR scan even if we have native nodes - scanNodes(view, isHybrid = true) - } else { - scanNodes(view, isHybrid = false) - } + scanNodes(view) + return container } @Composable private fun TopBarUI(onClose: () -> Unit) { - val filePickerLauncher = androidx.activity.compose.rememberLauncherForActivityResult( - contract = androidx.activity.result.contract.ActivityResultContracts.GetContent() - ) { uri: android.net.Uri? -> - if (uri != null) { - TesseractEngine.importModel(context, uri) { success, msg -> - android.widget.Toast.makeText(context, msg, android.widget.Toast.LENGTH_SHORT).show() - } - } - } Row( modifier = Modifier @@ -192,65 +173,11 @@ class CopyTextOverlayManager( ) { Icon(Icons.Default.Close, contentDescription = "Exit Copy Mode", tint = ComposeColor.White) } - - Spacer(modifier = Modifier.weight(1f)) - - Box { - var showMenu by remember { mutableStateOf(false) } - IconButton( - onClick = { showMenu = true }, - modifier = Modifier - .background(ComposeColor.Black.copy(alpha = 0.35f), CircleShape) - .size(40.dp) - ) { - Icon(Icons.Default.MoreVert, contentDescription = "Menu", tint = ComposeColor.White) - } - - DropdownMenu( - expanded = showMenu, - onDismissRequest = { showMenu = false }, - shape = RoundedCornerShape(28.dp), - tonalElevation = 6.dp - ) { - DropdownMenuItem( - text = { Text("Select Language / Model") }, - onClick = { - showMenu = false - showLanguageModelSelector() - } - ) - DropdownMenuItem( - text = { Text("Import Model (.traineddata)") }, - onClick = { - showMenu = false - filePickerLauncher.launch("*/*") - } - ) - } - } } } private fun Int.toComposeColor(): ComposeColor = ComposeColor(this) - private fun showLanguageModelSelector() { - val models = TesseractEngine.getAvailableModels(context) - val prefs = context.getSharedPreferences("OcrSettings", Context.MODE_PRIVATE) - val current = prefs.getString("selected_lang", "eng") ?: "eng" - - android.app.AlertDialog.Builder(context) - .setTitle("Select OCR Model") - .setSingleChoiceItems(models.toTypedArray(), models.indexOf(current)) { dialog, which -> - val selected = models[which] - prefs.edit().putString("selected_lang", selected).apply() - Toast.makeText(context, "Selected: ${selected.uppercase()}. Restarting scan...", Toast.LENGTH_SHORT).show() - dialog.dismiss() - rescanNodes() - } - .setNegativeButton("Cancel", null) - .show() - } - fun dismiss() { scanJob?.cancel() dimView = null @@ -262,41 +189,30 @@ class CopyTextOverlayManager( dimView?.let { scanNodes(it) } } - private fun scanNodes(view: View, isHybrid: Boolean = false) { + private fun scanNodes(view: View) { scanJob?.cancel() - statusMessage.value = null scanJob = scope.launch(Dispatchers.Main) { isScanning.value = true val bitmap = screenshotBitmap ?: BitmapRepository.getScreenshot() - + if (bitmap == null) { - if (isAssistMode && textNodes.isEmpty()) { - statusMessage.value = "This app doesn't allow reading screen content." - } isScanning.value = false view.invalidate() return@launch } try { - // OCR scan runs on background thread - val ocrNodes = TesseractEngine.extractText(context, bitmap) - - if (isHybrid) { - mergeHybridNodes(ocrNodes) - } else { - val sortedNodes = ocrNodes.sortedWith(compareBy({ it.bounds.top }, { it.bounds.left })) - textNodes.clear() - textNodes.addAll(sortedNodes) - updateAllWords() + val ocrNodes = withContext(Dispatchers.IO) { + com.akslabs.circletosearch.ocr.TesseractEngine.extractText(context, bitmap) } + val sortedNodes = ocrNodes.sortedWith(compareBy({ it.bounds.top }, { it.bounds.left })) + textNodes.clear() + textNodes.addAll(sortedNodes) + updateAllWords() - if (textNodes.isEmpty()) { - statusMessage.value = "No text found on screen." - } - - Log.d("CopyTextOverlay", "Capture complete: ${textNodes.size} total nodes") - } catch (e: Exception) { + onAnalysisCompleteCallback?.invoke(textNodes.size) + Log.d("CopyTextOverlay", "Analysis complete: ${textNodes.size} total nodes") + } catch (e: Throwable) { // Catch Throwable to prevent silent OutOfMemoryErrors from locking UI Log.e("CopyTextOverlay", "Extraction failed: ${e.message}") } finally { isScanning.value = false @@ -305,56 +221,15 @@ class CopyTextOverlayManager( } } - private fun mergeHybridNodes(ocrNodes: List) { - val newNodes = mutableListOf() - // Start with existing native nodes - newNodes.addAll(nativeNodes) - - for (ocr in ocrNodes) { - if (!isDuplicate(ocr, nativeNodes)) { - // Only add if OCR node is within screen bounds (roughly) - if (ocr.bounds.left >= -50 && ocr.bounds.top >= -50) { - newNodes.add(ocr) - } - } - } - - val sortedNodes = newNodes.sortedWith(compareBy({ it.bounds.top }, { it.bounds.left })) - textNodes.clear() - textNodes.addAll(sortedNodes) - updateAllWords() - } - - private fun isDuplicate(ocr: TextNode, natives: List): Boolean { - for (native in natives) { - // 1. Coordinate Overlap Check (> 50% overlap) - val ocrRect = ocr.bounds - val nativeRect = native.bounds - - val intersect = Rect(ocrRect) - if (intersect.intersect(nativeRect)) { - val intersectArea = intersect.width() * intersect.height() - val ocrArea = ocrRect.width() * ocrRect.height() - if (intersectArea > ocrArea * 0.5) return true - } - - // 2. Text Similarity Check - val ocrT = ocr.fullText.lowercase().trim() - val nativeT = native.fullText.lowercase().trim() - if (nativeT.contains(ocrT) || ocrT.contains(nativeT)) return true - } - return false - } - @SuppressLint("ClickableViewAccessibility") inner class DimPunchOutView(context: Context) : View(context) { private val density = resources.displayMetrics.density - private val viewLocation = IntArray(2) private val dimPaint = Paint().apply { color = Color.BLACK; alpha = 38; isAntiAlias = false } private val selectedWordPaint = Paint().apply { color = try { context.getColor(android.R.color.system_accent1_200) } catch(e: Exception) { Color.parseColor("#D0BCFF") } - alpha = 150; isAntiAlias = true + alpha = 90 + isAntiAlias = true } private val handlePaint = Paint().apply { color = Color.parseColor("#6750A4"); isAntiAlias = true } private val toolbarBgPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#F3EDF7") } @@ -374,77 +249,107 @@ class CopyTextOverlayManager( private var toolbarInitialized = false private var lastTouchX = 0f private var lastTouchY = 0f - - init { - setLayerType(LAYER_TYPE_SOFTWARE, null) + + // Allocation-free drawing fields + private val tempRect = RectF() + private val highlightPath = Path() + private val encompassingRect = RectF() + private var currentSx = 1f + private var currentSy = 1f + + private fun updateSelection(start: Int, end: Int) { + globalSelectionStart = start + globalSelectionEnd = end + recalculateHighlightPath() + invalidate() } - private fun toLocal(screenRect: Rect): RectF { - getLocationOnScreen(viewLocation) - return RectF( - (screenRect.left - viewLocation[0]).toFloat(), - (screenRect.top - viewLocation[1]).toFloat(), - (screenRect.right - viewLocation[0]).toFloat(), - (screenRect.bottom - viewLocation[1]).toFloat() - ) + private fun recalculateHighlightPath() { + highlightPath.reset() + encompassingRect.setEmpty() + + if (globalSelectionStart == -1 || globalSelectionEnd == -1) return + + val start = globalSelectionStart.coerceAtMost(globalSelectionEnd) + val end = globalSelectionStart.coerceAtLeast(globalSelectionEnd) + val selectedWords = (start..end).mapNotNull { allWords.getOrNull(it) } + + if (selectedWords.isEmpty()) return + + encompassingRect.set(selectedWords.first().bounds) + selectedWords.forEach { encompassingRect.union(it.bounds) } + + val sx = if (width > 0) width.toFloat() / (screenshotBitmap?.width?.toFloat() ?: 1f) else 1f + val sy = if (height > 0) height.toFloat() / (screenshotBitmap?.height?.toFloat() ?: 1f) else 1f + + selectedWords.groupBy { (it.bounds.centerY() / 20).toInt() }.forEach { (_, wordsInLine) -> + if (wordsInLine.isEmpty()) return@forEach + val first = wordsInLine.minByOrNull { it.bounds.left } ?: return@forEach + val last = wordsInLine.maxByOrNull { it.bounds.right } ?: return@forEach + val lineRect = RectF(first.bounds.left, wordsInLine.minOf { it.bounds.top }, last.bounds.right, wordsInLine.maxOf { it.bounds.bottom }) + lineRect.inset(-8f / sx, -4f / sy) + highlightPath.addRoundRect(lineRect, 8f / sx, 8f / sy, Path.Direction.CW) + } } - private fun toLocal(screenRectF: RectF): RectF { - getLocationOnScreen(viewLocation) - return RectF( - screenRectF.left - viewLocation[0], - screenRectF.top - viewLocation[1], - screenRectF.right - viewLocation[0], - screenRectF.bottom - viewLocation[1] - ) + init { + setLayerType(LAYER_TYPE_SOFTWARE, null) } - private fun toScreenX(localX: Float): Float { getLocationOnScreen(viewLocation); return localX + viewLocation[0] } - private fun toScreenY(localY: Float): Float { getLocationOnScreen(viewLocation); return localY + viewLocation[1] } + private fun scaleRect(r: RectF): RectF { + val sx = width.toFloat() / (screenshotBitmap?.width?.toFloat() ?: 1f) + val sy = height.toFloat() / (screenshotBitmap?.height?.toFloat() ?: 1f) + return RectF(r.left * sx, r.top * sy, r.right * sx, r.bottom * sy) + } + + private fun scaleRect(r: Rect): RectF = scaleRect(RectF(r)) override fun onDraw(canvas: Canvas) { + if (globalSelectionStart == -1 || globalSelectionEnd == -1) { + return // Do not dim or show text blocks if nothing is selected + } + val saveCount = canvas.saveLayer(0f, 0f, width.toFloat(), height.toFloat(), null) canvas.drawRect(0f, 0f, width.toFloat(), height.toFloat(), dimPaint) + // Use Matrix scaling to avoid allocating RectFs for every word + val sx = width.toFloat() / (screenshotBitmap?.width?.toFloat() ?: 1f) + val sy = height.toFloat() / (screenshotBitmap?.height?.toFloat() ?: 1f) + + if (sx != currentSx || sy != currentSy) { + currentSx = sx + currentSy = sy + recalculateHighlightPath() + } + + canvas.scale(sx, sy) + textNodes.forEach { node -> - val localBounds = toLocal(node.bounds) - localBounds.inset(-12f, -8f) - canvas.drawRoundRect(localBounds, 12f, 12f, clearPaint) + tempRect.set(node.bounds) + tempRect.inset(-8f / sx, -4f / sy) + canvas.drawRoundRect(tempRect, 8f / sx, 8f / sy, clearPaint) } if (globalSelectionStart != -1 && globalSelectionEnd != -1) { val start = globalSelectionStart.coerceAtMost(globalSelectionEnd) val end = globalSelectionStart.coerceAtLeast(globalSelectionEnd) - val selectedWords = (start..end).mapNotNull { allWords.getOrNull(it) } - - val highlightPath = Path() - selectedWords.groupBy { (it.bounds.centerY() / 20).toInt() }.forEach { (_, wordsInLine) -> - if (wordsInLine.isEmpty()) return@forEach - val first = wordsInLine.minBy { it.bounds.left } - val last = wordsInLine.maxBy { it.bounds.right } - val lineRect = RectF(first.bounds.left, wordsInLine.minOf { it.bounds.top }, last.bounds.right, wordsInLine.maxOf { it.bounds.bottom }) - val localHighlight = toLocal(lineRect) - localHighlight.inset(-12f, -8f) - highlightPath.addRoundRect(localHighlight, 8f, 8f, Path.Direction.CW) - } canvas.drawPath(highlightPath, selectedWordPaint) - val startLocal = toLocal(allWords[start].bounds) - val endLocal = toLocal(allWords[end].bounds) - drawHandle(canvas, startLocal.left, startLocal.top, true) - drawHandle(canvas, endLocal.right, endLocal.bottom, false) + drawHandle(canvas, allWords[start].bounds.left, allWords[start].bounds.top, true, sx, sy) + drawHandle(canvas, allWords[end].bounds.right, allWords[end].bounds.bottom, false, sx, sy) - val encompassing = RectF(allWords[start].bounds) - selectedWords.forEach { encompassing.union(it.bounds) } - drawFloatingToolbar(canvas, toLocal(encompassing)) + // Re-scale canvas to identity for UI components so they don't get stretched + canvas.scale(1/sx, 1/sy) + tempRect.set(encompassingRect.left * sx, encompassingRect.top * sy, encompassingRect.right * sx, encompassingRect.bottom * sy) + drawFloatingToolbar(canvas, tempRect) } canvas.restoreToCount(saveCount) } - private fun drawHandle(canvas: Canvas, x: Float, y: Float, isStart: Boolean) { - canvas.drawCircle(x, y, 18f, handlePaint) - if (isStart) canvas.drawRect(x - 2f, y, x + 2f, y + 40f, handlePaint) - else canvas.drawRect(x - 2f, y - 40f, x + 2f, y, handlePaint) + private fun drawHandle(canvas: Canvas, x: Float, y: Float, isStart: Boolean, sx: Float = 1f, sy: Float = 1f) { + canvas.drawCircle(x, y, 18f / sx, handlePaint) + if (isStart) canvas.drawRect(x - 2f / sx, y, x + 2f / sx, y + 40f / sy, handlePaint) + else canvas.drawRect(x - 2f / sx, y - 40f / sy, x + 2f / sx, y, handlePaint) } private fun drawFloatingToolbar(canvas: Canvas, anchor: RectF) { @@ -511,33 +416,33 @@ class CopyTextOverlayManager( if (globalSelectionStart != -1) { val start = globalSelectionStart.coerceAtMost(globalSelectionEnd) val end = globalSelectionStart.coerceAtLeast(globalSelectionEnd) - val startLocal = toLocal(allWords[start].bounds) - val endLocal = toLocal(allWords[end].bounds) + val startLocal = scaleRect(allWords[start].bounds) + val endLocal = scaleRect(allWords[end].bounds) if (isPointNear(lx, ly, startLocal.left, startLocal.top)) { dragHandleType = 1; return true } if (isPointNear(lx, ly, endLocal.right, endLocal.bottom)) { dragHandleType = 2; return true } } - val sx = toScreenX(lx); val sy = toScreenY(ly) - val nearest = findNearestWordGlobal(sx, sy) + val nearest = findNearestWordGlobal(lx, ly) if (nearest != -1) { - globalSelectionStart = nearest; globalSelectionEnd = nearest - performHapticFeedback(HapticFeedbackConstants.LONG_PRESS); invalidate(); return true - } else { globalSelectionStart = -1; globalSelectionEnd = -1; invalidate() } + updateSelection(nearest, nearest) + performHapticFeedback(HapticFeedbackConstants.LONG_PRESS); return true + } else { + updateSelection(-1, -1) + return false + } } MotionEvent.ACTION_MOVE -> { val dx = lx - lastTouchX; val dy = ly - lastTouchY lastTouchX = lx; lastTouchY = ly if (isDraggingToolbar) { toolbarOffsetX += dx; toolbarOffsetY += dy; invalidate(); return true } - val sx = toScreenX(lx); val sy = toScreenY(ly) if (dragHandleType != 0) { - val nearest = findNearestWordGlobal(sx, sy) + val nearest = findNearestWordGlobal(lx, ly) if (nearest != -1) { - if (dragHandleType == 1) globalSelectionStart = nearest else globalSelectionEnd = nearest - invalidate() + if (dragHandleType == 1) updateSelection(nearest, globalSelectionEnd) else updateSelection(globalSelectionStart, nearest) } return true } else if (globalSelectionStart != -1) { - val nearest = findNearestWordGlobal(sx, sy) - if (nearest != -1) { globalSelectionEnd = nearest; invalidate(); return true } + val nearest = findNearestWordGlobal(lx, ly) + if (nearest != -1) { updateSelection(globalSelectionStart, nearest); return true } } } MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { @@ -548,13 +453,12 @@ class CopyTextOverlayManager( } private fun findNearestWordGlobal(sx: Float, sy: Float): Int { - var minDist = Float.MAX_VALUE; var nearest = -1 allWords.forEachIndexed { idx, word -> - val dx = sx - word.bounds.centerX(); val dy = sy - word.bounds.centerY() - val d = dx * dx + dy * dy - if (d < minDist) { minDist = d; nearest = idx } + val scaled = scaleRect(word.bounds) + val expanded = RectF(scaled).apply { inset(-30f, -30f) } + if (expanded.contains(sx, sy)) return idx } - return if (minDist < 600 * 600) nearest else -1 + return -1 } private fun isPointNear(px: Float, py: Float, x: Float, y: Float): Boolean { diff --git a/app/src/main/java/com/akslabs/circletosearch/ui/components/SmartEntity.kt b/app/src/main/java/com/akslabs/circletosearch/ui/components/SmartEntity.kt new file mode 100644 index 0000000..d96222e --- /dev/null +++ b/app/src/main/java/com/akslabs/circletosearch/ui/components/SmartEntity.kt @@ -0,0 +1,23 @@ +package com.akslabs.circletosearch.ui.components + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Email +import androidx.compose.material.icons.filled.Link +import androidx.compose.material.icons.filled.Person +import androidx.compose.material.icons.filled.Phone + +// --- Phase 44: Smart Entity Extractor Models --- +sealed class SmartEntity( + val text: String, + val bounds: android.graphics.RectF, + val typeName: String, + val icon: ImageVector, + val sourceColor: Color +) { + class Url(text: String, bounds: android.graphics.RectF) : SmartEntity(text, bounds, "Link", Icons.Default.Link, Color(0xFF1A73E8)) + class Email(text: String, bounds: android.graphics.RectF) : SmartEntity(text, bounds, "Email", Icons.Default.Email, Color(0xFF1A73E8)) + class Phone(text: String, bounds: android.graphics.RectF) : SmartEntity(text, bounds, "Phone", Icons.Default.Phone, Color(0xFF43A047)) + class Upi(text: String, bounds: android.graphics.RectF) : SmartEntity(text, bounds, "UPI", Icons.Default.Person, Color(0xFF8E24AA)) +} \ No newline at end of file diff --git a/app/src/main/java/com/akslabs/circletosearch/ui/components/TranslationLanguageDialog.kt b/app/src/main/java/com/akslabs/circletosearch/ui/components/TranslationLanguageDialog.kt new file mode 100644 index 0000000..7812578 --- /dev/null +++ b/app/src/main/java/com/akslabs/circletosearch/ui/components/TranslationLanguageDialog.kt @@ -0,0 +1,281 @@ +/* + * + * * Copyright (C) 2025 AKS-Labs (original author) + * * + * * This program is free software: you can redistribute it and/or modify + * * it under the terms of the GNU General Public License as published by + * * the Free Software Foundation, either version 3 of the License, or + * * (at your option) any later version. + * * + * * This program is distributed in the hope that it will be useful, + * * but WITHOUT ANY WARRANTY; without even the implied warranty of + * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * * GNU General Public License for more details. + * * + * * You should have received a copy of the GNU General Public License + * * along with this program. If not, see . + * + */ + +package com.akslabs.circletosearch.ui.components + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Check +import androidx.compose.material.icons.filled.Language +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import com.google.mlkit.nl.translate.TranslateLanguage + +/** + * Pair: display name → ML Kit language code + */ +private data class LanguageOption( + val displayName: String, + val code: String, + val nativeName: String +) + +/** + * Full list of languages supported by ML Kit Translation. + * Sorted alphabetically by display name. + */ +private val SUPPORTED_LANGUAGES = listOf( + LanguageOption("Afrikaans", "af", "Afrikaans"), + LanguageOption("Arabic", "ar", "العربية"), + LanguageOption("Bengali", "bn", "বাংলা"), + LanguageOption("Bulgarian", "bg", "Български"), + LanguageOption("Catalan", "ca", "Català"), + LanguageOption("Chinese (Simplified)", "zh", "简体中文"), + LanguageOption("Chinese (Traditional)", "zh-TW", "繁體中文"), + LanguageOption("Croatian", "hr", "Hrvatski"), + LanguageOption("Czech", "cs", "Čeština"), + LanguageOption("Danish", "da", "Dansk"), + LanguageOption("Dutch", "nl", "Nederlands"), + LanguageOption("English", "en", "English"), + LanguageOption("Estonian", "et", "Eesti"), + LanguageOption("Filipino", "fil", "Filipino"), + LanguageOption("Finnish", "fi", "Suomi"), + LanguageOption("French", "fr", "Français"), + LanguageOption("German", "de", "Deutsch"), + LanguageOption("Greek", "el", "Ελληνικά"), + LanguageOption("Gujarati", "gu", "ગુજરાતી"), + LanguageOption("Hebrew", "he", "עברית"), + LanguageOption("Hindi", "hi", "हिन्दी"), + LanguageOption("Hungarian", "hu", "Magyar"), + LanguageOption("Icelandic", "is", "Íslenska"), + LanguageOption("Indonesian", "id", "Bahasa Indonesia"), + LanguageOption("Irish", "ga", "Gaeilge"), + LanguageOption("Italian", "it", "Italiano"), + LanguageOption("Japanese", "ja", "日本語"), + LanguageOption("Kannada", "kn", "ಕನ್ನಡ"), + LanguageOption("Korean", "ko", "한국어"), + LanguageOption("Latvian", "lv", "Latviešu"), + LanguageOption("Lithuanian", "lt", "Lietuvių"), + LanguageOption("Malay", "ms", "Bahasa Melayu"), + LanguageOption("Malayalam", "ml", "മലയാളം"), + LanguageOption("Marathi", "mr", "मराठी"), + LanguageOption("Norwegian", "nb", "Norsk"), + LanguageOption("Persian", "fa", "فارسی"), + LanguageOption("Polish", "pl", "Polski"), + LanguageOption("Portuguese", "pt", "Português"), + LanguageOption("Punjabi", "pa", "ਪੰਜਾਬੀ"), + LanguageOption("Romanian", "ro", "Română"), + LanguageOption("Russian", "ru", "Русский"), + LanguageOption("Serbian", "sr", "Српски"), + LanguageOption("Slovak", "sk", "Slovenčina"), + LanguageOption("Slovenian", "sl", "Slovenščina"), + LanguageOption("Spanish", "es", "Español"), + LanguageOption("Swahili", "sw", "Kiswahili"), + LanguageOption("Swedish", "sv", "Svenska"), + LanguageOption("Tamil", "ta", "தமிழ்"), + LanguageOption("Telugu", "te", "తెలుగు"), + LanguageOption("Thai", "th", "ไทย"), + LanguageOption("Turkish", "tr", "Türkçe"), + LanguageOption("Ukrainian", "uk", "Українська"), + LanguageOption("Urdu", "ur", "اردو"), + LanguageOption("Vietnamese", "vi", "Tiếng Việt") +).sortedBy { it.displayName } + +/** + * Dialog for selecting the target translation language. + * Shows a searchable list of all ML Kit supported languages. + */ +@Composable +fun TranslationLanguageDialog( + currentLanguageCode: String?, + onLanguageSelected: (String) -> Unit, + onDismiss: () -> Unit +) { + var searchQuery by remember { mutableStateOf("") } + var selectedCode by remember { mutableStateOf(currentLanguageCode ?: "en") } + + val filteredLanguages = remember(searchQuery) { + if (searchQuery.isBlank()) { + SUPPORTED_LANGUAGES + } else { + val q = searchQuery.lowercase() + SUPPORTED_LANGUAGES.filter { + it.displayName.lowercase().contains(q) || + it.nativeName.lowercase().contains(q) || + it.code.lowercase().contains(q) + } + } + } + + AlertDialog( + onDismissRequest = onDismiss, + modifier = Modifier.fillMaxSize().padding(20.dp), + shape = RoundedCornerShape(24.dp), + icon = { + Icon( + imageVector = Icons.Default.Language, + contentDescription = null, + modifier = Modifier.size(32.dp), + tint = MaterialTheme.colorScheme.primary + ) + }, + title = { + Text( + text = "Translate to", + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold + ) + }, + text = { + Column( + modifier = Modifier.fillMaxSize() + ) { + // Search field + androidx.compose.material3.OutlinedTextField( + value = searchQuery, + onValueChange = { searchQuery = it }, + modifier = Modifier.fillMaxWidth(), + placeholder = { Text("Search language...") }, + singleLine = true, + shape = RoundedCornerShape(12.dp) + ) + + Spacer(modifier = Modifier.height(12.dp)) + + // Language list + LazyColumn( + modifier = Modifier.weight(1f, fill = false), + verticalArrangement = Arrangement.spacedBy(4.dp) + ) { + items(filteredLanguages, key = { it.code }) { lang -> + LanguageRow( + option = lang, + isSelected = selectedCode == lang.code, + onClick = { selectedCode = lang.code } + ) + } + } + + Spacer(modifier = Modifier.height(8.dp)) + + // Auto-detect info + Text( + text = "Auto-detect: the source language is identified automatically for each text block.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + }, + confirmButton = { + Button( + onClick = { onLanguageSelected(selectedCode) }, + shape = RoundedCornerShape(12.dp) + ) { + Text("Translate") + } + }, + dismissButton = { + TextButton(onClick = onDismiss) { + Text("Cancel") + } + } + ) +} + +@Composable +private fun LanguageRow( + option: LanguageOption, + isSelected: Boolean, + onClick: () -> Unit +) { + Row( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(12.dp)) + .background( + if (isSelected) MaterialTheme.colorScheme.primaryContainer.copy(alpha = 0.4f) + else Color.Transparent + ) + .clickable(onClick = onClick) + .padding(horizontal = 12.dp, vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Column(modifier = Modifier.weight(1f)) { + Text( + text = option.displayName, + style = MaterialTheme.typography.bodyLarge, + fontWeight = if (isSelected) FontWeight.Bold else FontWeight.Normal, + color = if (isSelected) MaterialTheme.colorScheme.primary + else MaterialTheme.colorScheme.onSurface + ) + Text( + text = option.nativeName, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + + if (isSelected) { + Box( + modifier = Modifier + .size(24.dp) + .background(MaterialTheme.colorScheme.primary, CircleShape), + contentAlignment = Alignment.Center + ) { + Icon( + imageVector = Icons.Default.Check, + contentDescription = "Selected", + tint = MaterialTheme.colorScheme.onPrimary, + modifier = Modifier.size(16.dp) + ) + } + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/akslabs/circletosearch/utils/UIPreferences.kt b/app/src/main/java/com/akslabs/circletosearch/utils/UIPreferences.kt index f06c2e8..34ca8d6 100644 --- a/app/src/main/java/com/akslabs/circletosearch/utils/UIPreferences.kt +++ b/app/src/main/java/com/akslabs/circletosearch/utils/UIPreferences.kt @@ -35,6 +35,7 @@ class UIPreferences(context: Context) { private const val KEY_SHOW_FRIENDLY_MESSAGES = "show_friendly_messages" private const val KEY_SEARCH_ENGINE_ORDER = "search_engine_order" private const val KEY_USE_GOOGLE_LENS_ONLY = "use_google_lens_only" + private const val KEY_TARGET_TRANSLATE_LANG = "target_translate_lang" } fun observeUseGoogleLensOnly(): Flow = callbackFlow { @@ -98,4 +99,12 @@ class UIPreferences(context: Context) { fun setSearchEngineOrder(order: String) { prefs.edit().putString(KEY_SEARCH_ENGINE_ORDER, order).apply() } + + fun getTargetTranslateLang(): String? { + return prefs.getString(KEY_TARGET_TRANSLATE_LANG, null) + } + + fun setTargetTranslateLang(langCode: String?) { + prefs.edit().putString(KEY_TARGET_TRANSLATE_LANG, langCode).apply() + } } diff --git a/claude_instructions.md b/claude_instructions.md new file mode 100644 index 0000000..568f8d5 --- /dev/null +++ b/claude_instructions.md @@ -0,0 +1,57 @@ +# System Prompt & Project Context for Claude + +## 🎭 Твоя роль +Ты — Staff/Senior Android Developer и эксперт по безопасности, оптимизации производительности и системным API Android. Твоя задача — помогать развивать open-source проект "Circle To Search", писать чистый, поддерживаемый и отказоустойчивый код. + +## 📱 О проекте +"Circle To Search" — это privacy-first (ориентированная на приватность) альтернатива Google Lens / Circle to Search. +Приложение работает локально на устройстве (on-device), поддерживает De-Googled прошивки (GrapheneOS, LineageOS) и не отправляет данные на серверы без явного действия пользователя. +Основные функции: захват экрана (Accessibility / MediaProjection), умное выделение текста (Tesseract / ML Kit OCR), поиск по картинке и нативный перевод экрана. + +## 🛠 Стек технологий +- **Язык:** Kotlin +- **UI:** Jetpack Compose (Material 3) +- **Асинхронность:** Kotlin Coroutines & Flow +- **ML / AI:** Google ML Kit (Text Recognition, Translation), Tesseract4Android +- **Системные API:** AccessibilityService, MediaProjection, WindowManager (SYSTEM_ALERT_WINDOW) + +## 🏗 Ключевые компоненты (Архитектура) +1. **`CircleToSearchAccessibilityService`**: Системное сердце приложения. Обрабатывает жесты, отрисовывает зоны триггеров через `WindowManager` (без Activity) и делает скриншоты (`takeScreenshot`). Должно быть максимально стабильным. +2. **`OverlayActivity`**: Прозрачное окно (`Theme.Translucent.NoTitleBar`), которое открывается поверх системы для рендеринга скриншота и интерфейса взаимодействия (Jetpack Compose). +3. **`ScreenTranslator`**: Изолированный пайплайн (Closeable) для распознавания, перевода и умной отрисовки переведенного текста (StaticLayout) поверх Bitmap. +4. **`BitmapRepository`**: In-memory хранилище для передачи скриншотов между Service и Activity без использования Intent (превышение лимита Parcelable). + +## 🚨 Строгие правила написания кода (Strict Guidelines) + +При генерации кода строго соблюдай следующие правила: + +### 1. Memory Safety (Работа с Bitmap) +- Картинки экрана (Bitmap) весят много (8-15 МБ). Всегда учитывай риск `OutOfMemoryError`. +- Используй `try/catch (e: OutOfMemoryError)` при вызове `bitmap.copy()` или `Bitmap.createBitmap()`. +- Явно вызывай `.recycle()`, когда созданные вручную Bitmap больше не нужны, если они не передаются в Compose. + +### 2. Многопоточность (Coroutines) +- Никогда не блокируй Main Thread (UI-поток). +- Вся работа с файлами, сетью и инициализацией ML — в `Dispatchers.IO`. +- Вся работа с математикой, отрисовкой (Canvas), обработкой пикселей (getPixel) и ML Kit распознаванием — **строго в `Dispatchers.Default`**. +- Используй `withContext(Dispatchers.Main)` только для обновления UI-стейта. + +### 3. Жизненный цикл (Lifecycle) и Утечки ресурсов +- Клиенты ML Kit, камеры, Tesseract и другие "тяжелые" объекты должны реализовывать интерфейс `Closeable` или гарантированно закрываться (например, через блок `.use { }`). +- Корутины, запущенные внутри `Service`, должны быть привязаны к `SupervisorJob()` и отменяться в `onDestroy()`. + +### 4. Privacy First & Отказоустойчивость +- Приложение должно работать без доступа к интернету. Сетевые запросы возможны только для загрузки моделей (ML Kit) или перехода в браузер, и они должны быть обернуты в `try/catch`. +- Если пользователь использует GrapheneOS без Play Services, код не должен крашиться — используй fallback-логику или показывай понятные ошибки (Toast/Snackbar). +- Не используй сторонние трекеры, аналитику или логирование в облако. + +### 5. UI / UX (Jetpack Compose) +- Интерфейс должен быть плавным и использовать элементы Material 3. +- Для сложных наложений поверх скриншота используй `Canvas` и методы рисования напрямую (так как это дает больший контроль над пикселями). + +--- + +**Формат ответов:** +1. Сначала кратко объясни логику своего решения. +2. Пиши готовый к production код, учитывая импорты. +3. Обязательно добавляй комментарии к сложным участкам (особенно алгоритмическим). \ No newline at end of file diff --git a/gradlew b/gradlew old mode 100644 new mode 100755