diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8cdfcf69..9b6f29d8 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -94,6 +94,7 @@ dependencies { implementation(project(":samples:gemini-video-metadata-creation")) implementation(project(":samples:gemini-image-chat")) implementation(project(":samples:gemini-hybrid")) + implementation(project(":samples:gemini-hybrid-v4")) testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) diff --git a/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt b/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt index 1fadf927..56e14ec1 100644 --- a/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt +++ b/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt @@ -32,6 +32,7 @@ import com.android.ai.samples.genai_image_description.GenAIImageDescriptionScree import com.android.ai.samples.genai_summarization.GenAISummarizationScreen import com.android.ai.samples.genai_writing_assistance.GenAIWritingAssistanceScreen import com.android.ai.samples.geminihybrid.GeminiHybridScreen +import com.android.ai.samples.geminihybridv4.GeminiHybridV4Screen import com.android.ai.samples.imagen.ui.ImagenScreen import com.android.ai.samples.imagenediting.ui.ImagenEditingScreen import com.android.ai.samples.magicselfie.ui.MagicSelfieScreen @@ -51,6 +52,16 @@ val sampleCatalog = listOf( keyArt = R.drawable.img_keyart_text, isFeatured = true, ), + SampleCatalogItem( + title = R.string.gemini_hybrid_v4_sample_list_title, + description = R.string.gemini_hybrid_v4_sample_list_description, + route = "GeminiHybridV4Screen", + sampleEntryScreen = { GeminiHybridV4Screen() }, + tags = listOf(SampleTags.GEMINI_NANO, SampleTags.GEMINI_FLASH, SampleTags.ML_KIT, SampleTags.FIREBASE), + needsFirebase = true, + keyArt = R.drawable.img_keyart_text, + isFeatured = false, + ), SampleCatalogItem( title = R.string.gemini_image_chat_list_title, description = R.string.gemini_image_chat_list_description, diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 19e449e6..b7c1e408 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -29,6 +29,8 @@ Conversational Image generation with Gemini 3 Pro Image Gemini Hybrid Inference with Firebase Hybrid SDK using either Gemini Nano on-device or Gemini Flash in the Cloud. + Gemini Hybrid v4 + Inference with Firebase Hybrid SDK (nano v4 version). Firebase Required This feature requires Firebase to be initialized. Close diff --git a/samples/gemini-hybrid-v4/.gitignore b/samples/gemini-hybrid-v4/.gitignore new file mode 100644 index 00000000..796b96d1 --- /dev/null +++ b/samples/gemini-hybrid-v4/.gitignore @@ -0,0 +1 @@ +/build diff --git a/samples/gemini-hybrid-v4/README.md b/samples/gemini-hybrid-v4/README.md new file mode 100644 index 00000000..650cc640 --- /dev/null +++ b/samples/gemini-hybrid-v4/README.md @@ -0,0 +1,28 @@ +# Gemini Hybrid Sample + +This sample is part of the [AI Sample Catalog](../../). To build and run this sample, you should clone the entire repository. + +## Description + +This sample demonstrates how to use the Firebase Hybrid SDK, utilizing both on-device (Gemini Nano via [ML Kit Prompt API](https://developers.google.com/ml-kit/genai/prompt/android)) and cloud-based models via the [Firebase AI Logic SDK](https://firebase.google.com/docs/ai-logic). + +The sample lets users generate generic user reviews for a hotel based on a few selected topics. + +
+Gemini Hybrid SDK in action +
+ +## How it works + +Here is how the model is instantiated to leverage hybrid inference: +```kotlin +val model = Firebase.ai(backend = GenerativeBackend.googleAI()) + .generativeModel( + "gemini-2.5-flash-lite", + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_ON_DEVICE) + ) + +val response = model.generateContent(prompt) +``` + +Read more about the [Firebase Hybrid SDK](https://firebase.google.com/docs/ai-logic/hybrid/android/get-started?api=dev) in the Firebase documentation. diff --git a/samples/gemini-hybrid-v4/build.gradle.kts b/samples/gemini-hybrid-v4/build.gradle.kts new file mode 100644 index 00000000..65abb15a --- /dev/null +++ b/samples/gemini-hybrid-v4/build.gradle.kts @@ -0,0 +1,77 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.jetbrains.kotlin.android) + alias(libs.plugins.jetbrains.kotlin.serialization) + alias(libs.plugins.ksp) + alias(libs.plugins.compose.compiler) +} + +android { + namespace = "com.android.ai.samples.geminihybridv4" + compileSdk = 36 + + buildFeatures { + compose = true + } + + defaultConfig { + minSdk = 26 + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro", + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + kotlinOptions { + jvmTarget = "17" + } +} + +dependencies { + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.appcompat) + implementation(libs.androidx.material3) + implementation(libs.androidx.activity.compose) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.material.icons.extended) + implementation(libs.hilt.android) + implementation(libs.hilt.navigation.compose) + implementation(libs.androidx.runtime.livedata) + implementation(libs.androidx.lifecycle.runtime.compose) + implementation(platform(libs.firebase.bom)) + implementation(libs.firebase.ai) + implementation(libs.firebase.ai.ondevice) + implementation(libs.kotlinx.serialization.json) + + implementation(project(":ui-component")) + debugImplementation(libs.ui.tooling) + ksp(libs.hilt.compiler) +} diff --git a/samples/gemini-hybrid-v4/consumer-rules.pro b/samples/gemini-hybrid-v4/consumer-rules.pro new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/samples/gemini-hybrid-v4/consumer-rules.pro @@ -0,0 +1 @@ + diff --git a/samples/gemini-hybrid-v4/gemini_hybrid.png b/samples/gemini-hybrid-v4/gemini_hybrid.png new file mode 100644 index 00000000..1324a668 Binary files /dev/null and b/samples/gemini-hybrid-v4/gemini_hybrid.png differ diff --git a/samples/gemini-hybrid-v4/proguard-rules.pro b/samples/gemini-hybrid-v4/proguard-rules.pro new file mode 100644 index 00000000..f1b42451 --- /dev/null +++ b/samples/gemini-hybrid-v4/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/samples/gemini-hybrid-v4/src/main/AndroidManifest.xml b/samples/gemini-hybrid-v4/src/main/AndroidManifest.xml new file mode 100644 index 00000000..cfe0da1d --- /dev/null +++ b/samples/gemini-hybrid-v4/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/Expense.kt b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/Expense.kt new file mode 100644 index 00000000..cad9c65b --- /dev/null +++ b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/Expense.kt @@ -0,0 +1,10 @@ +package com.android.ai.samples.geminihybridv4 + +import kotlinx.serialization.Serializable + +@Serializable +data class Expense( + val name: String, + val price: Double, + val inferenceMode: String = "" +) diff --git a/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4Screen.kt b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4Screen.kt new file mode 100644 index 00000000..868667ba --- /dev/null +++ b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4Screen.kt @@ -0,0 +1,488 @@ +/* + * Copyright 2025 The Android Open Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:OptIn(PublicPreviewAPI::class) + +package com.android.ai.samples.geminihybridv4 + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.BitmapFactory +import android.net.Uri +import android.util.Log +import androidx.activity.compose.LocalOnBackPressedDispatcherOwner +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.Image +import androidx.compose.material3.Button +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.TextButton +import androidx.compose.material3.HorizontalDivider +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.window.Dialog +import android.graphics.Bitmap +import androidx.compose.material.icons.filled.Image +import androidx.compose.material.icons.filled.ArrowDropDown +import androidx.compose.material.icons.filled.Cloud +import androidx.compose.material.icons.filled.Smartphone +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.CameraAlt +import androidx.compose.material.icons.automirrored.filled.ReceiptLong +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.FloatingActionButton +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import androidx.core.content.FileProvider +import androidx.hilt.navigation.compose.hiltViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.android.ai.theme.AISampleCatalogTheme +import com.android.ai.theme.surfaceContainerHighestLight +import com.android.ai.uicomponent.SampleDetailTopAppBar +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.OnDeviceModelOption +import com.google.firebase.ai.type.Content +import com.google.firebase.ai.type.asTextOrNull +import java.io.File +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun GeminiHybridV4Screen(viewModel: GeminiHybridV4ViewModel = hiltViewModel()) { + val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val context = LocalContext.current + val backDispatcher = LocalOnBackPressedDispatcherOwner.current?.onBackPressedDispatcher + var tempUri by remember { mutableStateOf(null) } + var showImageDialog by remember { mutableStateOf(false) } + + val cameraLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.TakePicture(), + onResult = { success -> + if (success) { + tempUri?.let { uri -> + try { + val bitmap = BitmapFactory.decodeStream(context.contentResolver.openInputStream(uri)) + bitmap?.let { viewModel.scanReceipt(it, uri.toString()) } + } catch (e: Exception) { + Log.e("GeminiHybridV4Screen", "Failed to load image", e) + } + } + } + } + ) + + fun launchCamera() { + try { + val uri = getTmpFileUri(context) + tempUri = uri + cameraLauncher.launch(uri) + } catch (e: Exception) { + Log.e("GeminiHybridV4Screen", "Failed to create temp file", e) + } + } + + val permissionLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted -> + if (isGranted) { + launchCamera() + } + } + + AISampleCatalogTheme { + Scaffold( + modifier = Modifier.fillMaxSize(), + topBar = { + SampleDetailTopAppBar( + sampleName = stringResource(R.string.gemini_hybrid_v4_title), + sampleDescription = stringResource(R.string.gemini_hybrid_v4_description), + sourceCodeUrl = "https://github.com/android/ai-samples/tree/main/samples/gemini-hybrid-v4", + onBackClick = { backDispatcher?.onBackPressed() }, + ) + }, + floatingActionButton = { + FloatingActionButton( + onClick = { + val permissionCheckResult = + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) + if (permissionCheckResult == PackageManager.PERMISSION_GRANTED) { + launchCamera() + } else { + permissionLauncher.launch(Manifest.permission.CAMERA) + } + }, + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary + ) { + if (uiState.isScanning) { + CircularProgressIndicator( + modifier = Modifier.size(24.dp), + color = MaterialTheme.colorScheme.onPrimary, + strokeWidth = 2.dp + ) + } else { + Icon(Icons.Default.CameraAlt, contentDescription = stringResource(R.string.scan_receipt)) + } + } + } + ) { innerPadding -> + Box( + Modifier + .padding(innerPadding) + .fillMaxSize() + .clip(RoundedCornerShape(40.dp)) + .background(color = surfaceContainerHighestLight) + .padding(top = 16.dp, start = 16.dp, end = 16.dp, bottom = 32.dp), + contentAlignment = Alignment.Center, + ) { + Column( + Modifier + .padding(top = 16.dp) + .widthIn(max = 646.dp) + .fillMaxHeight(), + ) { + // Model Status Card + Card( + modifier = Modifier.fillMaxWidth(), + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.secondaryContainer + ) + ) { + Row( + modifier = Modifier.padding(12.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + Icons.AutoMirrored.Filled.ReceiptLong, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSecondaryContainer + ) + Spacer(modifier = Modifier.size(12.dp)) + Column { + Text( + stringResource(R.string.hybrid_v4_ai_status), + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + Text( + uiState.modelStatus, + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + } + Spacer(modifier = Modifier.weight(1f)) + ModelOptionDropdown( + selectedOption = uiState.selectedModelOption, + onOptionSelected = viewModel::setModelOption + ) + } + } + + Spacer(modifier = Modifier.height(16.dp)) + + Text( + stringResource(R.string.section_title), + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold + ) + + Spacer(modifier = Modifier.height(8.dp)) + + if (uiState.expense == null) { + Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { + Text(stringResource(R.string.no_expenses), color = Color.Gray) + } + } else { + ExpenseResultUi( + expense = uiState.expense!!, + hasImage = uiState.receiptImageUri != null, + onShowImage = { showImageDialog = true } + ) + } + + if (showImageDialog && uiState.receiptImageUri != null) { + val imageUri = Uri.parse(uiState.receiptImageUri) + var bitmap by remember(imageUri) { mutableStateOf(null) } + LaunchedEffect(imageUri) { + withContext(Dispatchers.IO) { + try { + bitmap = BitmapFactory.decodeStream(context.contentResolver.openInputStream(imageUri)) + } catch (e: Exception) { + Log.e("GeminiHybridV4Screen", "Failed to load image for dialog", e) + } + } + } + + if (bitmap != null) { + Dialog(onDismissRequest = { showImageDialog = false }) { + Card( + modifier = Modifier + .fillMaxWidth() + .height(400.dp), + shape = RoundedCornerShape(16.dp), + ) { + Box(modifier = Modifier.fillMaxSize()) { + Image( + bitmap = bitmap!!.asImageBitmap(), + contentDescription = "Receipt Image", + modifier = Modifier.fillMaxSize(), + contentScale = ContentScale.Fit + ) + } + } + } + } else { + Dialog(onDismissRequest = { showImageDialog = false }) { + Box(modifier = Modifier.size(100.dp), contentAlignment = Alignment.Center) { + CircularProgressIndicator() + } + } + } + } + + if (uiState.errorMessage != null) { + Spacer(modifier = Modifier.height(8.dp)) + Text( + text = uiState.errorMessage!!, + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.bodySmall + ) + } + } + } + } + } +} + +@Composable +fun ExpenseResultUi( + expense: Expense, + hasImage: Boolean, + onShowImage: () -> Unit +) { + Card( + modifier = Modifier.fillMaxWidth(), + elevation = CardDefaults.cardElevation(defaultElevation = 4.dp) + ) { + Column( + modifier = Modifier + .padding(16.dp) + .fillMaxWidth(), + verticalArrangement = Arrangement.spacedBy(12.dp) + ) { + Text( + text = expense.name, + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Bold + ) + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically + ) { + Text( + text = "Total:", + style = MaterialTheme.typography.bodyMedium, + color = Color.Gray + ) + Text( + text = "$${String.format("%.2f", expense.price)}", + style = MaterialTheme.typography.headlineMedium, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.primary + ) + } + + if (expense.inferenceMode.isNotEmpty()) { + val isCloud = expense.inferenceMode.equals("Cloud", ignoreCase = true) + val icon = if (isCloud) Icons.Default.Cloud else Icons.Default.Smartphone + val tint = if (isCloud) MaterialTheme.colorScheme.secondary else MaterialTheme.colorScheme.tertiary + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(6.dp) + ) { + Icon( + imageVector = icon, + contentDescription = null, + tint = tint, + modifier = Modifier.size(18.dp) + ) + Text( + text = "Extracted via ${expense.inferenceMode}", + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.SemiBold, + color = tint + ) + } + } + + if (hasImage) { + Button( + onClick = onShowImage, + modifier = Modifier.fillMaxWidth() + ) { + Icon(Icons.Default.Image, contentDescription = null) + Spacer(modifier = Modifier.width(8.dp)) + Text("Show Receipt Image") + } + } + + HorizontalDivider(modifier = Modifier.padding(vertical = 8.dp)) + + Text( + text = "Split Check", + style = MaterialTheme.typography.titleMedium, + fontWeight = FontWeight.Bold + ) + + var expanded by remember { mutableStateOf(false) } + var numPeople by remember(expense) { mutableStateOf(1) } + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically + ) { + Text("Number of people:") + Box { + TextButton(onClick = { expanded = true }) { + Text("$numPeople") + Spacer(modifier = Modifier.width(4.dp)) + Icon(Icons.Default.ArrowDropDown, contentDescription = null) + } + DropdownMenu( + expanded = expanded, + onDismissRequest = { expanded = false } + ) { + (1..10).forEach { count -> + DropdownMenuItem( + text = { Text("$count") }, + onClick = { + numPeople = count + expanded = false + } + ) + } + } + } + } + + val perPersonAmount = expense.price / numPeople + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically + ) { + Text("Per person:") + Text( + text = "$${String.format("%.2f", perPersonAmount)}", + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.secondary + ) + } + } + } +} + +private fun getTmpFileUri(context: Context): Uri { + val tmpFile = File.createTempFile("tmp_image_file", ".jpg", context.cacheDir).apply { + createNewFile() + deleteOnExit() + } + return FileProvider.getUriForFile(context, "${context.packageName}.provider", tmpFile) +} + +@Composable +fun ModelOptionDropdown( + selectedOption: OnDeviceModelOption, + onOptionSelected: (OnDeviceModelOption) -> Unit, + modifier: Modifier = Modifier, +) { + var expanded by remember { mutableStateOf(false) } + val options = listOf( + OnDeviceModelOption.STABLE to stringResource(R.string.gemini_hybrid_option_stable), + OnDeviceModelOption.PREVIEW to stringResource(R.string.gemini_hybrid_option_preview), + OnDeviceModelOption.PREVIEW_FAST to stringResource(R.string.gemini_hybrid_option_preview_fast), + ) + val selectedText = options.find { it.first == selectedOption }?.second ?: "" + + Box(modifier = modifier) { + TextButton(onClick = { expanded = true }) { + Text(selectedText) + Spacer(modifier = Modifier.width(4.dp)) + Icon( + imageVector = Icons.Default.ArrowDropDown, + contentDescription = null + ) + } + DropdownMenu( + expanded = expanded, + onDismissRequest = { expanded = false } + ) { + options.forEach { (option, label) -> + DropdownMenuItem( + text = { Text(label) }, + onClick = { + onOptionSelected(option) + expanded = false + } + ) + } + } + } +} diff --git a/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4ViewModel.kt b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4ViewModel.kt new file mode 100644 index 00000000..6917606c --- /dev/null +++ b/samples/gemini-hybrid-v4/src/main/java/com/android/ai/samples/geminihybridv4/GeminiHybridV4ViewModel.kt @@ -0,0 +1,187 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:OptIn(PublicPreviewAPI::class) + +package com.android.ai.samples.geminihybridv4 + +import android.graphics.Bitmap +import android.util.Log +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.google.firebase.Firebase +import com.google.firebase.ai.DownloadStatus +import com.google.firebase.ai.InferenceMode +import com.google.firebase.ai.InferenceSource +import com.google.firebase.ai.OnDeviceConfig +import com.google.firebase.ai.OnDeviceModelOption +import com.google.firebase.ai.OnDeviceModelStatus +import com.google.firebase.ai.ai +import com.google.firebase.ai.type.GenerativeBackend +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.content +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.Job +import kotlinx.serialization.json.Json + +data class GeminiHybridV4UiState( + val expense: Expense? = null, + val isScanning: Boolean = false, + val modelStatus: String = "Checking model status...", + val errorMessage: String? = null, + val receiptImageUri: String? = null, + val selectedModelOption: OnDeviceModelOption = OnDeviceModelOption.PREVIEW +) + +@OptIn(PublicPreviewAPI::class) +@HiltViewModel +class GeminiHybridV4ViewModel @Inject constructor() : ViewModel() { + private val _uiState = MutableStateFlow( + GeminiHybridV4UiState() + ) + val uiState: StateFlow = _uiState.asStateFlow() + + private fun getModel(option: OnDeviceModelOption) = Firebase.ai(backend = GenerativeBackend.googleAI()).generativeModel( + modelName = "gemini-3.1-flash-lite", + onDeviceConfig = OnDeviceConfig( + mode = InferenceMode.PREFER_IN_CLOUD, + modelOption = option) + ) + + init { + checkAndDownloadModel(_uiState.value.selectedModelOption) + } + + private var downloadJob: Job? = null + + private fun checkAndDownloadModel(option: OnDeviceModelOption) { + downloadJob?.cancel() + downloadJob = viewModelScope.launch { + val model = getModel(option) + try { + val status = model.onDeviceExtension?.checkStatus() ?: return@launch + + updateStatus(status) + + if (status == OnDeviceModelStatus.DOWNLOADABLE) { + model.onDeviceExtension?.download()?.collect { downloadStatus -> + when (downloadStatus) { + is DownloadStatus.DownloadStarted -> { + _uiState.update { it.copy(modelStatus = "Downloading model...") } + } + + is DownloadStatus.DownloadInProgress -> { + val progress = downloadStatus.totalBytesDownloaded + _uiState.update { it.copy(modelStatus = "Downloading: $progress bytes downloaded") } + } + + is DownloadStatus.DownloadCompleted -> { + _uiState.update { it.copy(modelStatus = "Model ready") } + } + + is DownloadStatus.DownloadFailed -> { + _uiState.update { + it.copy( + modelStatus = "Download failed", errorMessage = "Model download failed" + ) + } + } + } + } + } + } catch (e: Exception) { + _uiState.update { it.copy(modelStatus = "Error checking status", errorMessage = e.message) } + } + } + } + + private fun updateStatus(status: OnDeviceModelStatus) { + val statusText = when (status) { + OnDeviceModelStatus.AVAILABLE -> "Model available" + OnDeviceModelStatus.DOWNLOADABLE -> "Model downloadable" + OnDeviceModelStatus.DOWNLOADING -> "Model downloading..." + OnDeviceModelStatus.UNAVAILABLE -> "On-device model unavailable" + else -> "Unknown" + } + _uiState.update { it.copy(modelStatus = statusText) } + } + + fun setModelOption(option: OnDeviceModelOption) { + if (_uiState.value.selectedModelOption != option) { + _uiState.update { it.copy(selectedModelOption = option) } + checkAndDownloadModel(option) + } + } + + fun scanReceipt(bitmap: Bitmap, uriString: String) { + viewModelScope.launch { + _uiState.update { it.copy(isScanning = true, errorMessage = null, receiptImageUri = uriString) } + try { + val prompt = content { + image(bitmap) + text( + """ + Extract the store name and the total price from this receipt. Store names are usually at the top of the document, first thing in them. Total price is usually indicated by text like "Total" or "Total Due" + Output only in JSON format containg 2 fields '{name,price}'. + Do not include any currency signs or backticks or any text around it. + Use dots for decimals. + Examples: + - {"name": "FakeStore", "price": "2.0"} + - {"name": "SomeMarket", "price": "3.5"} + """.trimIndent() + ) + } + + val model = getModel(_uiState.value.selectedModelOption) + val response = model.generateContent(prompt) + val text = response.text + val inferenceMode = if (response.inferenceSource == InferenceSource.ON_DEVICE) { + "On-device" + } else { + "Cloud" + } + Log.d("HybridVM", "$inferenceMode response: $text") + if (text != null) { + parseAndAddExpense(text, inferenceMode) + } else { + _uiState.update { it.copy(errorMessage = "Could not extract data") } + } + } catch (e: Exception) { + _uiState.update { it.copy(errorMessage = "Error: ${e.message}") } + } finally { + _uiState.update { it.copy(isScanning = false) } + } + } + } + + private fun parseAndAddExpense(text: String, inferenceMode: String) { + val json = text + // The on-device model sometimes outputs backticks, so we remove those + .replace("```json", "") + .replace("```", "") + try { + val newExpense = Json.decodeFromString(json).copy(inferenceMode = inferenceMode) + _uiState.update { it.copy(expense = newExpense) } + } catch (e: Exception) { + _uiState.update { it.copy(errorMessage = e.localizedMessage) } + } + } +} diff --git a/samples/gemini-hybrid-v4/src/main/res/values/strings.xml b/samples/gemini-hybrid-v4/src/main/res/values/strings.xml new file mode 100644 index 00000000..655644ba --- /dev/null +++ b/samples/gemini-hybrid-v4/src/main/res/values/strings.xml @@ -0,0 +1,12 @@ + + + Hybrid Inference (Receipt Scanner) + Scan receipts and extract expenses using either Gemini Nano on-device or Gemini Flash in the Cloud. + Hybrid AI Status + Scanned Receipts + No expenses yet. Scan a receipt to add one. + Scan Receipt + STABLE + PREVIEW + PREVIEW FAST + diff --git a/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridScreen.kt b/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridScreen.kt index a15fb129..8ed8562a 100644 --- a/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridScreen.kt +++ b/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridScreen.kt @@ -19,9 +19,11 @@ package com.android.ai.samples.geminihybrid import androidx.activity.compose.LocalOnBackPressedDispatcherOwner import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.ExperimentalLayoutApi +import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.FlowRow import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxHeight @@ -77,6 +79,7 @@ import com.android.ai.uicomponent.GenerateButton import com.android.ai.uicomponent.SampleDetailTopAppBar import com.android.ai.uicomponent.UndoButton import com.google.firebase.ai.InferenceMode +import com.google.firebase.ai.OnDeviceModelOption import com.google.firebase.ai.type.PublicPreviewAPI @@ -135,6 +138,8 @@ fun GeminiHybridScreen(viewModel: GeminiHybridViewModel = hiltViewModel()) { onTagToggle = viewModel::toggleTag, selectedMode = uiState.selectedMode, onModeSelected = viewModel::setInferenceMode, + selectedModelOption = uiState.selectedModelOption, + onModelOptionSelected = viewModel::setModelOption, onGenerate = { val tagStrings = uiState.selectedTags.map { ContextCompat.getString(context, it) } @@ -185,6 +190,8 @@ fun InitialReviewUi( onTagToggle: (Int) -> Unit, selectedMode: InferenceMode, onModeSelected: (InferenceMode) -> Unit, + selectedModelOption: OnDeviceModelOption, + onModelOptionSelected: (OnDeviceModelOption) -> Unit, onGenerate: () -> Unit, ) { Text( @@ -215,10 +222,27 @@ fun InitialReviewUi( } } Spacer(Modifier.height(50.dp)) - InferenceModeDropdown( - selectedMode = selectedMode, - onModeSelected = onModeSelected, - ) + Row( + modifier = Modifier + .fillMaxWidth() + .padding(start = 8.dp, top = 12.dp, end = 8.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + Box(modifier = Modifier.weight(1f)) { + InferenceModeDropdown( + selectedMode = selectedMode, + onModeSelected = onModeSelected, + modifier = Modifier.fillMaxWidth() + ) + } + Box(modifier = Modifier.weight(1f)) { + ModelOptionDropdown( + selectedOption = selectedModelOption, + onOptionSelected = onModelOptionSelected, + modifier = Modifier.fillMaxWidth() + ) + } + } GenerateButton( text = stringResource(R.string.gemini_hybrid_generate_btn), @@ -402,6 +426,7 @@ fun LanguageDropdown( fun InferenceModeDropdown( selectedMode: InferenceMode, onModeSelected: (InferenceMode) -> Unit, + modifier: Modifier = Modifier, ) { var expanded by remember { mutableStateOf(false) } val modes = listOf( @@ -412,7 +437,7 @@ fun InferenceModeDropdown( ) val selectedText = modes.find { it.first == selectedMode }?.second ?: "" - Box(modifier = Modifier.padding(start = 8.dp, top = 12.dp)) { + Box(modifier = modifier) { SplitButtonLayout( leadingButton = { SplitButtonDefaults.LeadingButton( @@ -457,6 +482,67 @@ fun InferenceModeDropdown( } } +@PublicPreviewAPI +@Composable +fun ModelOptionDropdown( + selectedOption: OnDeviceModelOption, + onOptionSelected: (OnDeviceModelOption) -> Unit, + modifier: Modifier = Modifier, +) { + var expanded by remember { mutableStateOf(false) } + val options = listOf( + OnDeviceModelOption.STABLE to stringResource(R.string.gemini_hybrid_option_stable), + OnDeviceModelOption.PREVIEW to stringResource(R.string.gemini_hybrid_option_preview), + OnDeviceModelOption.PREVIEW_FAST to stringResource(R.string.gemini_hybrid_option_preview_fast), + ) + val selectedText = options.find { it.first == selectedOption }?.second ?: "" + + Box(modifier = modifier) { + SplitButtonLayout( + leadingButton = { + SplitButtonDefaults.LeadingButton( + onClick = { expanded = true }, + colors = ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.tertiaryContainer, + contentColor = MaterialTheme.colorScheme.onTertiaryContainer, + ), + ) { + Text(selectedText) + } + }, + trailingButton = { + SplitButtonDefaults.TrailingButton( + onClick = { expanded = true }, + colors = ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.tertiaryContainer, + contentColor = MaterialTheme.colorScheme.onTertiaryContainer, + ), + ) { + Icon( + imageVector = Icons.Default.ArrowDropDown, + contentDescription = null, + ) + } + }, + ) + DropdownMenu( + expanded = expanded, + onDismissRequest = { expanded = false }, + ) { + options.forEach { (option, label) -> + DropdownMenuItem( + text = { Text(label) }, + onClick = { + onOptionSelected(option) + expanded = false + }, + ) + } + } + } +} + + @Composable fun StatusText(text: String) { Text( diff --git a/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridViewModel.kt b/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridViewModel.kt index 240e99a5..37e6bc6d 100644 --- a/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridViewModel.kt +++ b/samples/gemini-hybrid/src/main/java/com/android/ai/samples/geminihybrid/GeminiHybridViewModel.kt @@ -58,6 +58,7 @@ data class GeminiHybridUiState( val reviewText: String = "", val reviewInferenceStatus: Int? = null, val selectedLanguage: String = "Korean", + val selectedModelOption: OnDeviceModelOption = OnDeviceModelOption.STABLE, val status: GeminiStatus = GeminiStatus.Initial ) @@ -88,6 +89,10 @@ class GeminiHybridViewModel @Inject constructor() : ViewModel() { _uiState.update { it.copy(selectedMode = mode) } } + fun setModelOption(option: OnDeviceModelOption) { + _uiState.update { it.copy(selectedModelOption = option) } + } + fun toggleTag(tagResId: Int) { _uiState.update { state -> val newTags = if (state.selectedTags.contains(tagResId)) { @@ -133,7 +138,7 @@ class GeminiHybridViewModel @Inject constructor() : ViewModel() { "gemini-2.5-flash-lite", onDeviceConfig = OnDeviceConfig( mode = _uiState.value.selectedMode, - modelOption = OnDeviceModelOption.STABLE) + modelOption = _uiState.value.selectedModelOption) ) model.onDeviceExtension?.checkStatus() model.generateContentStream(prompt).collect { chunk -> diff --git a/samples/gemini-hybrid/src/main/res/values/strings.xml b/samples/gemini-hybrid/src/main/res/values/strings.xml index 82a3fc9e..d8313289 100644 --- a/samples/gemini-hybrid/src/main/res/values/strings.xml +++ b/samples/gemini-hybrid/src/main/res/values/strings.xml @@ -25,4 +25,7 @@ SPACIOUS NATURAL LIGHT Select topics for your review: + STABLE + PREVIEW + PREVIEW FAST diff --git a/settings.gradle.kts b/settings.gradle.kts index c21b7a73..8619decd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,4 +52,5 @@ include(":samples:gemini-live-todo") include(":samples:gemini-video-metadata-creation") include(":samples:gemini-image-chat") include(":samples:gemini-hybrid") +include(":samples:gemini-hybrid-v4") include(":ui-component")