Add Android sample (JNI binding + Java wrapper class)

CMakeLists.txt

@@ -180,7 +180,12 @@ if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES
         # TODO: arm msvc?
     else()
         if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-            add_compile_options(-mcpu=native)
+            # Check CLIP_NATIVE here to add -mcpu=native flag
+            # Without this check, Android NDK complains that '-mcpu=native'
+            # is not supported even after setting -DCLIP_NATIVE=Off
+            if (CLIP_NATIVE)
+                add_compile_options(-mcpu=native)
+            endif()
         endif()
         # TODO: armv6,7,8 version specific flags
     endif()

clip.cpp

@@ -536,9 +536,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
 
         int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN);
         int idx_std = get_key_idx(ctx, KEY_IMAGE_STD);
+
+        // `gguf_get_arr_data(ctx, idx_mean) + i` is needed here, to access all elements from 
+        // entities with ids `idx_mean` and `idx_std`. 
+        // See issue https://github.com/monatis/clip.cpp/issues/99
         for (int i = 0; i < 3; ++i) {
-            new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean));
-            new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std));
+            new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean) + i);
+            new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std) + i);
         }
 
         if (verbosity >= 2) {
@@ -795,6 +799,35 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip
     return true;
 }
 
+bool clip_image_preprocess_no_resize(const clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res) {
+    if (!ctx->has_vision_encoder) {
+        printf("This gguf file seems to have no vision encoder\n");
+        return false;
+    }
+    const int nx2 = ctx->vision_model.hparams.image_size;
+    const int ny2 = ctx->vision_model.hparams.image_size;
+
+    res->nx = nx2;
+    res->ny = ny2;
+    res->size = 3 * nx2 * ny2;
+    res->data = new float[res->size]();
+
+    const auto & m3 = ctx->image_mean; // {0.48145466f, 0.4578275f, 0.40821073f};
+    const auto & s3 = ctx->image_std;  // {0.26862954f, 0.26130258f, 0.27577711f};
+
+    for (int y = 0; y < ny2; y++) {
+        for (int x = 0; x < nx2; x++) {
+            for (int c = 0; c < 3; c++) {
+                const int i = 3 * (y * nx2 + x) + c;
+                res->data[i] = ((float(img->data[i]) / 255.0f) - m3[c]) / s3[c];
+            }
+        }
+    }
+
+    return true;
+
+}
+
 // Structure to hold the image data as an input to function to be executed for thread
 typedef struct {
     const clip_image_u8 * input;

clip.h

@@ -87,6 +87,8 @@ void clip_image_f32_free(struct clip_image_f32 * res);
 bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
 bool clip_image_preprocess(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res);
 
+bool clip_image_preprocess_no_resize(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res);
+
 bool clip_text_encode(const struct clip_ctx * ctx, const int n_threads, const struct clip_tokens * tokens, float * vec,
                       const bool normalize);
 bool clip_image_encode(const struct clip_ctx * ctx, const int n_threads, struct clip_image_f32 * img, float * vec,

examples/clip.android/.gitignore

@@ -0,0 +1,10 @@
+*.iml
+.gradle
+/local.properties
+/.idea
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

examples/clip.android/README.md

@@ -0,0 +1,50 @@
+# clip.cpp - Android App
+
+An Android app demonstrating the usage of `clip.cpp` library. It uses JNI and a Java wrapper class to interface with the functions provided in `clip.h`. 
+
+## Setup
+
+### Build the app
+
+1. Open the current directory (`clip.cpp/examples/clip.android`) in Android Studio. An automatic Gradle build should start, if not click on the `Build` menu and select `Make Project`.
+
+2. Connect the test-device to the computer and make sure that the device is recognized by the computer.
+
+3. Download one of the GGUF models from the [HuggingFace repository](https://huggingface.co/my). For instance, if we download the `CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf` model, we need to push it to the test-device's file-system using `adb push`,
+
+```commandline
+adb push CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf /data/local/tmp/clip_model_fp16.gguf
+```
+
+4. In `MainActivityViewModel.kt`, ensure that the `modelPath` variable points to the correct model path on the test-device. For instance, if the model is pushed to `/data/local/tmp/clip_model_fp16.gguf`, then the `modelPath` variable should be set to `/data/local/tmp/clip_model_fp16.gguf`. Moreover, you can configure `NUM_THREADS` and `VERBOSITY` variables as well. 
+
+```kotlin
+private val MODEL_PATH = "/data/local/tmp/clip_model_fp16.gguf"
+private val NUM_THREADS = 4
+private val VERBOSITY = 1
+```
+
+5. Run the app on the test-device by clicking on the `Run` button (Shift + F10) in Android Studio.
+
+### Run tests
+
+This Android project also includes an instrumented test which would require an Android device (emulator or physical device). 
+
+1. Open the current directory (`clip.cpp/examples/clip.android`) in Android Studio. An automatic Gradle build should start, if not click on the `Build` menu and select `Make Project`.
+
+2. Connect the test-device to the computer and make sure that the device is recognized by the computer.
+
+3. Download one of the GGUF models from the [HuggingFace repository](https://huggingface.co/my). For instance, if we download the `CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf` model, we need to push it to the test-device's file-system using `adb push`,
+
+```commandline
+adb push CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf /data/local/tmp/clip_model.gguf
+```
+
+4. Get two images from the internet and push them to the test-device's file-system using `adb push`,
+
+```commandline
+adb push image1.png /data/local/tmp/sample.png
+adb push image2.png /data/local/tmp/sample_2.png
+```
+
+5. Navigate to `clip.cpp/examples/clip.android/clip/src/androidTest/java/android/example/clip/CLIPAndroidInstrumentedTest.kt`, right-click on the file, select `Run 'CLIPAndroidInstrumentedTest'` from the context menu.

examples/clip.android/app/.gitignore

@@ -0,0 +1 @@
+/build

examples/clip.android/app/build.gradle.kts

@@ -0,0 +1,71 @@
+plugins {
+    alias(libs.plugins.android.application)
+    alias(libs.plugins.jetbrains.kotlin.android)
+}
+
+android {
+    namespace = "android.example.clip"
+    compileSdk = 34
+
+    defaultConfig {
+        applicationId = "android.example.clip"
+        minSdk = 26
+        targetSdk = 34
+        versionCode = 1
+        versionName = "1.0"
+
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+        vectorDrawables {
+            useSupportLibrary = true
+        }
+    }
+
+    buildTypes {
+        release {
+            isMinifyEnabled = false
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+    buildFeatures {
+        compose = true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion = "1.5.1"
+    }
+    packaging {
+        resources {
+            excludes += "/META-INF/{AL2.0,LGPL2.1}"
+        }
+    }
+}
+
+dependencies {
+    implementation(libs.androidx.core.ktx)
+    implementation(libs.androidx.lifecycle.runtime.ktx)
+    implementation(libs.androidx.activity.compose)
+    implementation(platform(libs.androidx.compose.bom))
+    implementation(libs.androidx.ui)
+    implementation(libs.androidx.ui.graphics)
+    implementation(libs.androidx.ui.tooling.preview)
+    implementation(libs.androidx.material3)
+    implementation("androidx.exifinterface:exifinterface:1.3.7" )
+    implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.5")
+    implementation(project(":clip"))
+    testImplementation(libs.junit)
+    androidTestImplementation(libs.androidx.junit)
+    androidTestImplementation(libs.androidx.espresso.core)
+    androidTestImplementation(platform(libs.androidx.compose.bom))
+    androidTestImplementation(libs.androidx.ui.test.junit4)
+    debugImplementation(libs.androidx.ui.tooling)
+    debugImplementation(libs.androidx.ui.test.manifest)
+}

examples/clip.android/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

examples/clip.android/app/src/main/AndroidManifest.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.Clipcpp"
+        tools:targetApi="31">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@style/Theme.Clipcpp">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>