diff --git a/CMakeLists.txt b/CMakeLists.txt index bca4bdb..b3c6980 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,7 +180,12 @@ if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES # TODO: arm msvc? else() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") - add_compile_options(-mcpu=native) + # Check CLIP_NATIVE here to add -mcpu=native flag + # Without this check, Android NDK complains that '-mcpu=native' + # is not supported even after setting -DCLIP_NATIVE=Off + if (CLIP_NATIVE) + add_compile_options(-mcpu=native) + endif() endif() # TODO: armv6,7,8 version specific flags endif() diff --git a/clip.cpp b/clip.cpp index a3710de..5ceb677 100644 --- a/clip.cpp +++ b/clip.cpp @@ -536,9 +536,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN); int idx_std = get_key_idx(ctx, KEY_IMAGE_STD); + + // `gguf_get_arr_data(ctx, idx_mean) + i` is needed here, to access all elements from + // entities with ids `idx_mean` and `idx_std`. + // See issue https://github.com/monatis/clip.cpp/issues/99 for (int i = 0; i < 3; ++i) { - new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean)); - new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std)); + new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean) + i); + new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std) + i); } if (verbosity >= 2) { @@ -795,6 +799,35 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip return true; } +bool clip_image_preprocess_no_resize(const clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res) { + if (!ctx->has_vision_encoder) { + printf("This gguf file seems to have no vision encoder\n"); + return false; + } + const int nx2 = ctx->vision_model.hparams.image_size; + const int ny2 = ctx->vision_model.hparams.image_size; + + res->nx = nx2; + res->ny = ny2; + res->size = 3 * nx2 * ny2; + res->data = new float[res->size](); + + const auto & m3 = ctx->image_mean; // {0.48145466f, 0.4578275f, 0.40821073f}; + const auto & s3 = ctx->image_std; // {0.26862954f, 0.26130258f, 0.27577711f}; + + for (int y = 0; y < ny2; y++) { + for (int x = 0; x < nx2; x++) { + for (int c = 0; c < 3; c++) { + const int i = 3 * (y * nx2 + x) + c; + res->data[i] = ((float(img->data[i]) / 255.0f) - m3[c]) / s3[c]; + } + } + } + + return true; + +} + // Structure to hold the image data as an input to function to be executed for thread typedef struct { const clip_image_u8 * input; diff --git a/clip.h b/clip.h index 183b22d..0ebefe0 100644 --- a/clip.h +++ b/clip.h @@ -87,6 +87,8 @@ void clip_image_f32_free(struct clip_image_f32 * res); bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); bool clip_image_preprocess(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res); +bool clip_image_preprocess_no_resize(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res); + bool clip_text_encode(const struct clip_ctx * ctx, const int n_threads, const struct clip_tokens * tokens, float * vec, const bool normalize); bool clip_image_encode(const struct clip_ctx * ctx, const int n_threads, struct clip_image_f32 * img, float * vec, diff --git a/examples/clip.android/.gitignore b/examples/clip.android/.gitignore new file mode 100644 index 0000000..10cfdbf --- /dev/null +++ b/examples/clip.android/.gitignore @@ -0,0 +1,10 @@ +*.iml +.gradle +/local.properties +/.idea +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/examples/clip.android/README.md b/examples/clip.android/README.md new file mode 100644 index 0000000..76d9de9 --- /dev/null +++ b/examples/clip.android/README.md @@ -0,0 +1,50 @@ +# clip.cpp - Android App + +An Android app demonstrating the usage of `clip.cpp` library. It uses JNI and a Java wrapper class to interface with the functions provided in `clip.h`. + +## Setup + +### Build the app + +1. Open the current directory (`clip.cpp/examples/clip.android`) in Android Studio. An automatic Gradle build should start, if not click on the `Build` menu and select `Make Project`. + +2. Connect the test-device to the computer and make sure that the device is recognized by the computer. + +3. Download one of the GGUF models from the [HuggingFace repository](https://huggingface.co/my). For instance, if we download the `CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf` model, we need to push it to the test-device's file-system using `adb push`, + +```commandline +adb push CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf /data/local/tmp/clip_model_fp16.gguf +``` + +4. In `MainActivityViewModel.kt`, ensure that the `modelPath` variable points to the correct model path on the test-device. For instance, if the model is pushed to `/data/local/tmp/clip_model_fp16.gguf`, then the `modelPath` variable should be set to `/data/local/tmp/clip_model_fp16.gguf`. Moreover, you can configure `NUM_THREADS` and `VERBOSITY` variables as well. + +```kotlin +private val MODEL_PATH = "/data/local/tmp/clip_model_fp16.gguf" +private val NUM_THREADS = 4 +private val VERBOSITY = 1 +``` + +5. Run the app on the test-device by clicking on the `Run` button (Shift + F10) in Android Studio. + +### Run tests + +This Android project also includes an instrumented test which would require an Android device (emulator or physical device). + +1. Open the current directory (`clip.cpp/examples/clip.android`) in Android Studio. An automatic Gradle build should start, if not click on the `Build` menu and select `Make Project`. + +2. Connect the test-device to the computer and make sure that the device is recognized by the computer. + +3. Download one of the GGUF models from the [HuggingFace repository](https://huggingface.co/my). For instance, if we download the `CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf` model, we need to push it to the test-device's file-system using `adb push`, + +```commandline +adb push CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf /data/local/tmp/clip_model.gguf +``` + +4. Get two images from the internet and push them to the test-device's file-system using `adb push`, + +```commandline +adb push image1.png /data/local/tmp/sample.png +adb push image2.png /data/local/tmp/sample_2.png +``` + +5. Navigate to `clip.cpp/examples/clip.android/clip/src/androidTest/java/android/example/clip/CLIPAndroidInstrumentedTest.kt`, right-click on the file, select `Run 'CLIPAndroidInstrumentedTest'` from the context menu. diff --git a/examples/clip.android/app/.gitignore b/examples/clip.android/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/examples/clip.android/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/examples/clip.android/app/build.gradle.kts b/examples/clip.android/app/build.gradle.kts new file mode 100644 index 0000000..5e7d3fc --- /dev/null +++ b/examples/clip.android/app/build.gradle.kts @@ -0,0 +1,71 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.jetbrains.kotlin.android) +} + +android { + namespace = "android.example.clip" + compileSdk = 34 + + defaultConfig { + applicationId = "android.example.clip" + minSdk = 26 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + vectorDrawables { + useSupportLibrary = true + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + buildFeatures { + compose = true + } + composeOptions { + kotlinCompilerExtensionVersion = "1.5.1" + } + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } +} + +dependencies { + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.activity.compose) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.ui) + implementation(libs.androidx.ui.graphics) + implementation(libs.androidx.ui.tooling.preview) + implementation(libs.androidx.material3) + implementation("androidx.exifinterface:exifinterface:1.3.7" ) + implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.5") + implementation(project(":clip")) + testImplementation(libs.junit) + androidTestImplementation(libs.androidx.junit) + androidTestImplementation(libs.androidx.espresso.core) + androidTestImplementation(platform(libs.androidx.compose.bom)) + androidTestImplementation(libs.androidx.ui.test.junit4) + debugImplementation(libs.androidx.ui.tooling) + debugImplementation(libs.androidx.ui.test.manifest) +} \ No newline at end of file diff --git a/examples/clip.android/app/proguard-rules.pro b/examples/clip.android/app/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/examples/clip.android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/examples/clip.android/app/src/main/AndroidManifest.xml b/examples/clip.android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..3bb958e --- /dev/null +++ b/examples/clip.android/app/src/main/AndroidManifest.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/java/android/example/clip/MainActivity.kt b/examples/clip.android/app/src/main/java/android/example/clip/MainActivity.kt new file mode 100644 index 0000000..02c78c8 --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/MainActivity.kt @@ -0,0 +1,314 @@ +/* + * MIT License + * + * Copyright (c) 2024 Shubham Panchal + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package android.example.clip + +import android.example.clip.ui.components.AppProgressDialog +import android.example.clip.ui.components.hideProgressDialog +import android.example.clip.ui.components.setProgressDialogText +import android.example.clip.ui.components.showProgressDialog +import android.example.clip.ui.theme.ClipcppTheme +import android.graphics.Bitmap +import android.graphics.BitmapFactory.* +import android.graphics.Matrix +import androidx.exifinterface.media.ExifInterface +import android.net.Uri +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.compose.setContent +import androidx.activity.enableEdgeToEdge +import androidx.activity.result.PickVisualMediaRequest +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.Image +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.ColumnScope +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Add +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.Info +import androidx.compose.material3.Button +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.TextField +import androidx.compose.material3.TopAppBar +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.window.Dialog +import androidx.lifecycle.viewmodel.compose.viewModel + +class MainActivity : ComponentActivity() { + + @OptIn(ExperimentalMaterial3Api::class) + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + enableEdgeToEdge() + setContent { + + val viewModel = viewModel() + + ClipcppTheme { + Scaffold( + modifier = Modifier.fillMaxSize(), + topBar = { + TopAppBar( + title = { Text(text = "clip.android") }, + actions = { + Row { + IconButton(onClick = { + viewModel.showModelInfo() + }) { + Icon( + imageVector = Icons.Default.Info, + contentDescription = "Model Info" + ) + } + } + } + ) + } + ) { innerPadding -> + Column(modifier = Modifier.padding(innerPadding)) { + SelectImagePanel(viewModel) + EnterDescriptionPanel(viewModel) + } + LoadModelProgressDialog(viewModel) + RunningInferenceProgressDialog(viewModel) + ModelInfoDialog(viewModel) + } + } + } + } + + @Composable + private fun ColumnScope.SelectImagePanel(viewModel: MainActivityViewModel) { + var selectedImage by remember { viewModel.selectedImageState } + val pickMediaLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.PickVisualMedia() + ) { + if (it != null) { + val bitmap = getFixedBitmap(it) + selectedImage = bitmap + } + } + Column( + modifier = Modifier + .fillMaxSize() + .background(Color.LightGray) + .weight(1f), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + if (selectedImage == null) { + Button(modifier = Modifier.padding(vertical = 40.dp), onClick = { + pickMediaLauncher.launch( + PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly) + ) + }) { + Icon(imageVector = Icons.Default.Add, contentDescription = "Select an image") + Text(text = "Select an image") + } + } else { + Image( + bitmap = selectedImage!!.asImageBitmap(), + contentDescription = "Selected image", + modifier = Modifier.fillMaxWidth() + ) + } + } + } + + @Composable + private fun ColumnScope.EnterDescriptionPanel(viewModel: MainActivityViewModel) { + var description by remember{ viewModel.descriptionState } + val similarityScore by remember{ viewModel.similarityScoreState } + Column( + modifier = Modifier + .fillMaxSize() + .weight(1f) + ) { + Column( + modifier = Modifier + .fillMaxSize() + .padding(24.dp), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + if (similarityScore == null) { + TextField( + modifier = Modifier + .fillMaxWidth() + .padding(vertical = 16.dp), + label = { Text(text = "Enter a description") }, + value = description, + onValueChange = { description = it } + ) + Button( + enabled = description.isNotEmpty(), + onClick = { viewModel.compare() } + ) { + Text(text = "Compare") + } + } + else { + Text( + text = "Similarity score: $similarityScore", + modifier = Modifier.padding(vertical = 16.dp), + fontSize = 24.sp + ) + Button(onClick = { viewModel.reset() }) { + Text(text = "Compare again") + } + } + } + } + } + + @Composable + private fun LoadModelProgressDialog(viewModel: MainActivityViewModel) { + val isLoadingModel by remember{ viewModel.isLoadingModelState } + if (isLoadingModel) { + showProgressDialog() + setProgressDialogText("Loading model...") + } + else { + hideProgressDialog() + } + AppProgressDialog() + } + + @Composable + private fun RunningInferenceProgressDialog(viewModel: MainActivityViewModel) { + val isInferenceRunning by remember{ viewModel.isInferenceRunning } + if (isInferenceRunning) { + showProgressDialog() + setProgressDialogText("Running inference...") + } + else { + hideProgressDialog() + } + AppProgressDialog() + } + + @Composable + private fun ModelInfoDialog(viewModel: MainActivityViewModel) { + var showDialog by remember{ viewModel.isShowingModelInfoDialogState } + if (showDialog && viewModel.visionHyperParameters != null && viewModel.textHyperParameters != null) { + Dialog( + onDismissRequest = { showDialog = false } + ) { + Column( + modifier = Modifier + .fillMaxWidth() + .padding(8.dp) + .background(Color.White, RoundedCornerShape(8.dp)) + .padding(16.dp) + ) { + Row( + verticalAlignment = Alignment.CenterVertically + ) { + Text( + text = "Model Info", + modifier = Modifier.weight(1f), + style = MaterialTheme.typography.headlineLarge + ) + IconButton( + onClick = { showDialog = false } + ) { + Icon( + imageVector = Icons.Default.Close, + contentDescription = "Close Model Info Dialog" + ) + } + } + Spacer(modifier = Modifier.height(8.dp)) + + Text(text = "Vision Hyper-parameters", style = MaterialTheme.typography.bodyLarge) + Spacer(modifier = Modifier.height(4.dp)) + Text(text = "imageSize = ${viewModel.visionHyperParameters?.imageSize}", style = MaterialTheme.typography.labelSmall) + Text(text = "hiddenSize = ${viewModel.visionHyperParameters?.hiddenSize}", style = MaterialTheme.typography.labelSmall) + Text(text = "patchSize = ${viewModel.visionHyperParameters?.patchSize}", style = MaterialTheme.typography.labelSmall) + Text(text = "projectionDim = ${viewModel.visionHyperParameters?.projectionDim}", style = MaterialTheme.typography.labelSmall) + Text(text = "num layers = ${viewModel.visionHyperParameters?.nLayer}", style = MaterialTheme.typography.labelSmall) + Text(text = "num intermediate = ${viewModel.visionHyperParameters?.nIntermediate}", style = MaterialTheme.typography.labelSmall) + Text(text = "num heads = ${viewModel.visionHyperParameters?.nHead}", style = MaterialTheme.typography.labelSmall) + Spacer(modifier = Modifier.height(8.dp)) + + Text(text = "Text Hyper-parameters", style = MaterialTheme.typography.bodyLarge) + Spacer(modifier = Modifier.height(4.dp)) + Text(text = "num positions = ${viewModel.textHyperParameters?.numPositions}", style = MaterialTheme.typography.labelSmall) + Text(text = "hiddenSize = ${viewModel.textHyperParameters?.hiddenSize}", style = MaterialTheme.typography.labelSmall) + Text(text = "num vocab = ${viewModel.textHyperParameters?.nVocab}", style = MaterialTheme.typography.labelSmall) + Text(text = "projectionDim = ${viewModel.textHyperParameters?.projectionDim}", style = MaterialTheme.typography.labelSmall) + Text(text = "num layers = ${viewModel.textHyperParameters?.nLayer}", style = MaterialTheme.typography.labelSmall) + Text(text = "num intermediate = ${viewModel.textHyperParameters?.nIntermediate}", style = MaterialTheme.typography.labelSmall) + Text(text = "num heads = ${viewModel.textHyperParameters?.nHead}", style = MaterialTheme.typography.labelSmall) + } + } + } + } + + private fun getFixedBitmap(imageFileUri: Uri): Bitmap { + var imageBitmap = decodeStream(contentResolver.openInputStream(imageFileUri)) + val exifInterface = ExifInterface(contentResolver.openInputStream(imageFileUri)!!) + imageBitmap = when (exifInterface.getAttributeInt( + ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_UNDEFINED + )) { + ExifInterface.ORIENTATION_ROTATE_90 -> rotateBitmap(imageBitmap, 90f) + ExifInterface.ORIENTATION_ROTATE_180 -> rotateBitmap(imageBitmap, 180f) + ExifInterface.ORIENTATION_ROTATE_270 -> rotateBitmap(imageBitmap, 270f) + else -> imageBitmap + } + return imageBitmap + } + + private fun rotateBitmap(source: Bitmap, degrees: Float): Bitmap { + val matrix = Matrix() + matrix.postRotate(degrees) + return Bitmap.createBitmap(source, 0, 0, source.width, source.height, matrix, false) + } + +} + diff --git a/examples/clip.android/app/src/main/java/android/example/clip/MainActivityViewModel.kt b/examples/clip.android/app/src/main/java/android/example/clip/MainActivityViewModel.kt new file mode 100644 index 0000000..e5ff4a6 --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/MainActivityViewModel.kt @@ -0,0 +1,127 @@ +/* + * MIT License + * + * Copyright (c) 2024 Shubham Panchal + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package android.example.clip + +import android.clip.cpp.CLIPAndroid +import android.graphics.Bitmap +import androidx.compose.runtime.mutableStateOf +import androidx.lifecycle.ViewModel +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.nio.ByteBuffer + +class MainActivityViewModel: ViewModel() { + + val selectedImageState = mutableStateOf(null) + val descriptionState = mutableStateOf("") + val isLoadingModelState = mutableStateOf(true) + val isInferenceRunning = mutableStateOf(false) + val isShowingModelInfoDialogState = mutableStateOf(false) + val similarityScoreState = mutableStateOf(null) + private val clipAndroid = CLIPAndroid() + var visionHyperParameters: CLIPAndroid.CLIPVisionHyperParameters? = null + var textHyperParameters: CLIPAndroid.CLIPTextHyperParameters? = null + + private val MODEL_PATH = "/data/local/tmp/clip_model_fp16.gguf" + private val NUM_THREADS = 4 + private val VERBOSITY = 1 + + init { + CoroutineScope(Dispatchers.IO).launch { + mainScope { isLoadingModelState.value = true } + clipAndroid.load(MODEL_PATH, VERBOSITY) + visionHyperParameters = clipAndroid.visionHyperParameters + textHyperParameters = clipAndroid.textHyperParameters + mainScope { isLoadingModelState.value = false } + } + } + + fun compare() { + if (selectedImageState.value != null && descriptionState.value.isNotEmpty()) { + CoroutineScope(Dispatchers.Default).launch { + mainScope { isInferenceRunning.value = true } + val textEmbedding = clipAndroid.encodeText( + descriptionState.value, + NUM_THREADS, + textHyperParameters?.projectionDim ?: 512, + true + ) + val imageBuffer = bitmapToByteBuffer(selectedImageState.value!!) + val imageEmbedding = clipAndroid.encodeImage( + imageBuffer, + selectedImageState.value!!.width, + selectedImageState.value!!.height, + NUM_THREADS, + visionHyperParameters?.projectionDim ?: 512, + true + ) + mainScope { + similarityScoreState.value = clipAndroid.getSimilarityScore(textEmbedding, imageEmbedding) + isInferenceRunning.value = false + } + } + } + } + + fun showModelInfo() { + isShowingModelInfoDialogState.value = true + } + + fun reset() { + selectedImageState.value = null + descriptionState.value = "" + similarityScoreState.value = null + isInferenceRunning.value = false + } + + override fun onCleared() { + super.onCleared() + clipAndroid.close() + } + + private suspend fun mainScope(action: () -> Unit) { + withContext(Dispatchers.Main) { + action() + } + } + + private fun bitmapToByteBuffer(bitmap: Bitmap): ByteBuffer { + val width = bitmap.width + val height = bitmap.height + val imageBuffer = ByteBuffer.allocateDirect(width * height * 3) + for (y in 0 until height) { + for (x in 0 until width) { + val pixel = bitmap.getPixel(x, y) + imageBuffer.put((pixel shr 16 and 0xFF).toByte()) + imageBuffer.put((pixel shr 8 and 0xFF).toByte()) + imageBuffer.put((pixel and 0xFF).toByte()) + } + } + return imageBuffer + } + +} \ No newline at end of file diff --git a/examples/clip.android/app/src/main/java/android/example/clip/ui/components/AppProgressDialog.kt b/examples/clip.android/app/src/main/java/android/example/clip/ui/components/AppProgressDialog.kt new file mode 100644 index 0000000..80411ee --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/ui/components/AppProgressDialog.kt @@ -0,0 +1,65 @@ +package android.example.clip.ui.components + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog + +private val progressDialogVisibleState = mutableStateOf(false) +private val progressDialogText = mutableStateOf("") + +@Composable +fun AppProgressDialog() { + val isVisible by remember { progressDialogVisibleState } + if (isVisible) { + Dialog(onDismissRequest = { /* Progress dialogs are non-cancellable */ }) { + Box( + contentAlignment = Alignment.Center, + modifier = + Modifier.fillMaxWidth() + .background(Color.White, shape = RoundedCornerShape(8.dp)) + ) { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier.padding(vertical = 24.dp) + ) { + LinearProgressIndicator(modifier = Modifier.fillMaxWidth()) + Spacer(modifier = Modifier.padding(4.dp)) + Text( + text = progressDialogText.value, + textAlign = TextAlign.Center, + modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp) + ) + } + } + } + } +} + +fun setProgressDialogText(message: String) { + progressDialogText.value = message +} + +fun showProgressDialog() { + progressDialogVisibleState.value = true + progressDialogText.value = "" +} + +fun hideProgressDialog() { + progressDialogVisibleState.value = false +} \ No newline at end of file diff --git a/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Color.kt b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Color.kt new file mode 100644 index 0000000..3b494c2 --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Color.kt @@ -0,0 +1,11 @@ +package android.example.clip.ui.theme + +import androidx.compose.ui.graphics.Color + +val Purple80 = Color(0xFFD0BCFF) +val PurpleGrey80 = Color(0xFFCCC2DC) +val Pink80 = Color(0xFFEFB8C8) + +val Purple40 = Color(0xFF6650a4) +val PurpleGrey40 = Color(0xFF625b71) +val Pink40 = Color(0xFF7D5260) \ No newline at end of file diff --git a/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Theme.kt b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Theme.kt new file mode 100644 index 0000000..ad416a8 --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Theme.kt @@ -0,0 +1,58 @@ +package android.example.clip.ui.theme + +import android.app.Activity +import android.os.Build +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.platform.LocalContext + +private val DarkColorScheme = darkColorScheme( + primary = Purple80, + secondary = PurpleGrey80, + tertiary = Pink80 +) + +private val LightColorScheme = lightColorScheme( + primary = Purple40, + secondary = PurpleGrey40, + tertiary = Pink40 + + /* Other default colors to override + background = Color(0xFFFFFBFE), + surface = Color(0xFFFFFBFE), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFF1C1B1F), + onSurface = Color(0xFF1C1B1F), + */ +) + +@Composable +fun ClipcppTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + // Dynamic color is available on Android 12+ + dynamicColor: Boolean = true, + content: @Composable () -> Unit +) { + val colorScheme = when { + dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { + val context = LocalContext.current + if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + } + + darkTheme -> DarkColorScheme + else -> LightColorScheme + } + + MaterialTheme( + colorScheme = colorScheme, + typography = Typography, + content = content + ) +} \ No newline at end of file diff --git a/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Type.kt b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Type.kt new file mode 100644 index 0000000..eb7d0f3 --- /dev/null +++ b/examples/clip.android/app/src/main/java/android/example/clip/ui/theme/Type.kt @@ -0,0 +1,34 @@ +package android.example.clip.ui.theme + +import androidx.compose.material3.Typography +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +// Set of Material typography styles to start with +val Typography = Typography( + bodyLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 16.sp, + lineHeight = 24.sp, + letterSpacing = 0.5.sp + ) + /* Other default text styles to override + titleLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 22.sp, + lineHeight = 28.sp, + letterSpacing = 0.sp + ), + labelSmall = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Medium, + fontSize = 11.sp, + lineHeight = 16.sp, + letterSpacing = 0.5.sp + ) + */ +) \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/drawable/ic_launcher_background.xml b/examples/clip.android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/examples/clip.android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/clip.android/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/clip.android/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/examples/clip.android/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml b/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml new file mode 100644 index 0000000..6f3b755 --- /dev/null +++ b/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml b/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml new file mode 100644 index 0000000..6f3b755 --- /dev/null +++ b/examples/clip.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 0000000..c209e78 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 0000000..b2dfe3d Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 0000000..4f0f1d6 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp new file mode 100644 index 0000000..62b611d Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 0000000..948a307 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..1b9a695 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp new file mode 100644 index 0000000..28d4b77 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..9287f50 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp new file mode 100644 index 0000000..aa7d642 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..9126ae3 Binary files /dev/null and b/examples/clip.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/examples/clip.android/app/src/main/res/values/colors.xml b/examples/clip.android/app/src/main/res/values/colors.xml new file mode 100644 index 0000000..f8c6127 --- /dev/null +++ b/examples/clip.android/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/values/strings.xml b/examples/clip.android/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..02fb3c1 --- /dev/null +++ b/examples/clip.android/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + clip.cpp + \ No newline at end of file diff --git a/examples/clip.android/app/src/main/res/values/themes.xml b/examples/clip.android/app/src/main/res/values/themes.xml new file mode 100644 index 0000000..91da76d --- /dev/null +++ b/examples/clip.android/app/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + + +