Add Android QNN Browserstack test

java/src/test/android/README.md

@@ -23,12 +23,16 @@ This android application is mainly aimed for testing:
 - The [Gradle](https://gradle.org/) build system is required for building the APKs used to run [android instrumentation tests](https://source.android.com/compatibility/tests/development/instrumentation). Version 7.5 or newer is required.
   The Gradle wrapper at `java/gradlew[.bat]` may be used.
 
+
 ### Building
 
 Use the android's [build instructions](https://onnxruntime.ai/docs/build/android.html) with `--build_java` and `--android_run_emulator` option.
 
 Please note that you may need to set the `--android_abi=x86_64` (the default option is `arm64-v8a`). This is because android instrumentation test is run on an android emulator which requires an abi of `x86_64`.
 
+#### QNN Builds
+For QNN builds, it isnecessary to set the `ADSP_LIBRARY_PATH` environment variable to the [native library directory](https://developer.android.com/reference/android/content/pm/ApplicationInfo#nativeLibraryDir) depending on the device. This ensures that any native libraries downloaded as dependencies such as QNN libraries are found by the application.
+
 #### Build Output
 
 The build will generate two apks which is required to run the test application in `$YOUR_BUILD_DIR/java/androidtest/android/app/build/outputs/apk`:
@@ -37,3 +41,5 @@ The build will generate two apks which is required to run the test application i
 * `debug/app-debug.apk`
 
 After running the build script, the two apks will be installed on `ort_android` emulator and it will automatically run the test application in an adb shell.
+
+#### Manual Testing on QDC

java/src/test/android/app/build.gradle

@@ -4,6 +4,7 @@ plugins {
 }
 
 def minSdkVer = System.properties.get("minSdkVer")?:24
+def qnnVersion = System.properties['qnnVersion']
 
 android {
 	compileSdkVersion 32
@@ -16,6 +17,14 @@ android {
 		versionName "1.0"
 
 		testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+
+		 // Add BuildConfig field for qnnVersion
+		if (qnnVersion != null) {
+			buildConfigField "boolean", "IS_QNN_BUILD", "true"
+		}
+		else {
+			buildConfigField "boolean", "IS_QNN_BUILD", "false"
+		}
 	}
 
 	buildTypes {
@@ -31,6 +40,22 @@ android {
 	kotlinOptions {
 		jvmTarget = '1.8'
 	}
+	// Conditional packagingOptions for QNN builds only
+	if (qnnVersion != null)	{
+		packagingOptions  {
+			jniLibs {
+				useLegacyPackaging = true
+			}
+			exclude 'lib/arm64-v8a/libQnnGpu.so'
+		}
+	}
+	sourceSets {
+        main {
+            if (System.properties['qnnVersion'] != null) {
+                manifest.srcFile 'src/main/AndroidManifestQnn.xml'  // Use QNN manifest
+            }
+        }
+    }
 	namespace 'ai.onnxruntime.example.javavalidator'
 }
 
@@ -49,4 +74,10 @@ dependencies {
 	androidTestImplementation 'androidx.test:runner:1.4.0'
 	androidTestImplementation 'androidx.test:rules:1.4.0'
 	androidTestImplementation 'com.microsoft.appcenter:espresso-test-extension:1.4'
+
+	// QNN only dependencies
+	if (qnnVersion != null) {
+	implementation "com.qualcomm.qti:qnn-runtime:$qnnVersion"
+	}
+
 }

java/src/test/android/app/src/androidTest/java/ai/onnxruntime/example/javavalidator/SimpleTest.kt

@@ -44,7 +44,12 @@ class SimpleTest {
 
     @Test
     fun runSigmoidModelTestNNAPI() {
-        runSigmoidModelTestImpl(1, true)
+        runSigmoidModelTestImpl(1, useNNAPI = true)
+    }
+
+    @Test
+    fun runSigmoidModelTestQNN() {
+        runSigmoidModelTestImpl(1, useQNN = true)
     }
 
     @Throws(IOException::class)
@@ -54,14 +59,17 @@ class SimpleTest {
     }
 
     @Throws(OrtException::class, IOException::class)
-    fun runSigmoidModelTestImpl(intraOpNumThreads: Int, useNNAPI: Boolean = false) {
-        reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, useNNAPI=$useNNAPI")
+    fun runSigmoidModelTestImpl(intraOpNumThreads: Int, useNNAPI: Boolean = false, useQNN: Boolean = false) {
+        reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, useNNAPI=$useNNAPI, useQNN=$useQNN")
         Log.println(Log.INFO, TAG, "Testing with intraOpNumThreads=$intraOpNumThreads")
         Log.println(Log.INFO, TAG, "Testing with useNNAPI=$useNNAPI")
+        Log.println(Log.INFO, TAG, "Testing with useQNN=$useQNN")
+
         val env = OrtEnvironment.getEnvironment(OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE)
         env.use {
             val opts = SessionOptions()
             opts.setIntraOpNumThreads(intraOpNumThreads)
+
             if (useNNAPI) {
                 if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.NNAPI)) {
                     opts.addNnapi()
@@ -70,6 +78,18 @@ class SimpleTest {
                     return
                 }
             }
+
+            if (useQNN) {
+                if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.QNN)) {
+                    // Since this is running in an Android environment, we use the .so library
+                    val qnnLibrary = "libQnnHtp.so"
+                    val providerOptions = Collections.singletonMap("backend_path", qnnLibrary)
+
+                    opts.addQnn(providerOptions)
+                }
+            }
+
+
             opts.use {
                 val session = env.createSession(readModel("sigmoid.ort"), opts)
                 session.use {
@@ -92,13 +112,14 @@ class SimpleTest {
                         output.use {
                             @Suppress("UNCHECKED_CAST")
                             val rawOutput = output[0].value as Array<Array<FloatArray>>
+                            val precision = if (useQNN) 1e-3 else 1e-6
                             for (i in 0..2) {
                                 for (j in 0..3) {
                                     for (k in 0..4) {
                                         Assert.assertEquals(
                                             rawOutput[i][j][k],
                                             expected[i][j][k],
-                                            1e-6.toFloat()
+                                            precision.toFloat()
                                         )
                                     }
                                 }

java/src/test/android/app/src/main/AndroidManifest.xml

@@ -17,4 +17,4 @@
         </activity>
     </application>
 
-</manifest>
+</manifest>

java/src/test/android/app/src/main/AndroidManifestQnn.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.JavaValidator">
+        <activity android:name=".MainActivity" android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+        <uses-native-library
+        android:name="libcdsprpc.so"
+        android:required="false" />
+    </application>
+
+</manifest>

java/src/test/android/app/src/main/java/ai/onnxruntime/example/javavalidator/MainActivity.kt

@@ -1,11 +1,19 @@
 package ai.onnxruntime.example.javavalidator
 
 import android.os.Bundle
+import android.system.Os
 import androidx.appcompat.app.AppCompatActivity
 
 /*Empty activity app mainly used for testing*/
 class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
+        if (BuildConfig.IS_QNN_BUILD) {
+            val adspLibraryPath = applicationContext.applicationInfo.nativeLibraryDir
+            // set the path variable to the native library directory
+            // so that the any native libraries downloaded as dependencies
+            // (like qnn libs) are found
+            Os.setenv("ADSP_LIBRARY_PATH", adspLibraryPath, true)
+        }
         super.onCreate(savedInstanceState)
     }
-}
+}

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml

@@ -14,6 +14,11 @@ parameters:
   type: string
   default: 'onnxruntime-android'
 
+- name: QnnSDKVersion
+  displayName: QNN SDK version
+  type: string
+  default: '2.26.0.240828'
+
 jobs:
 - job: Final_AAR_Testing_Android_${{ parameters.job_name_suffix }}
   workspace:
@@ -45,38 +50,56 @@ jobs:
 
   - template: use-android-ndk.yml
 
-  - template: use-android-emulator.yml
-    parameters:
-      create: true
-      start: true
-
   - script: |
-      set -e -x
-      mkdir android_test
-      cd android_test
-      cp -av $(Build.SourcesDirectory)/java/src/test/android ./
-      cd ./android
-      mkdir -p app/libs
-      cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-android.aar
-      $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
-    displayName: Run E2E test using Emulator
+        set -e -x
+        mkdir -p android_test/android/app/libs
+        cd android_test/android
+        cp -av $(Build.SourcesDirectory)/java/src/test/android/* ./
+        cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-android.aar
+    displayName: Copy Android test files and AAR to android_test directory
     workingDirectory: $(Build.BinariesDirectory)
 
-  - template: use-android-emulator.yml
-    parameters:
-      stop: true
+  # skip emulator tests for qnn package as there are no arm64-v8a emulators
+  - ${{ if not(contains(parameters.packageName, 'qnn')) }}:
+    - template: use-android-emulator.yml
+      parameters:
+        create: true
+        start: true
+
+    - script: |
+        set -e -x
+        cd android_test/android
+        $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
+      displayName: Run E2E test using Emulator
+      workingDirectory: $(Build.BinariesDirectory)
+
+    - template: use-android-emulator.yml
+      parameters:
+        stop: true
 
   # we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23)
   - script: |
       set -e -x
+      cd android_test/android
+      # build apks for qnn package as they are not built in the emulator test step
+      if [[ ${{ parameters.packageName }} == *qnn* ]]; then
+          # qnn-runtime package is expected to be of the form qnn-runtime <major>.<minor>.<patch>
+          qnn_version=$(echo "${{ parameters.QnnSDKVersion }}" | cut -d'.' -f1-3)
+          $(Build.SourcesDirectory)/java/gradlew  --no-daemon clean assembleDebug assembleAndroidTest -DqnnVersion=$qnn_version --stacktrace
+      fi
+
       pip install requests
+      cd app/build/outputs/apk
+
       python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
         --test_platform espresso \
         --app_apk_path "debug/app-debug.apk" \
         --test_apk_path "androidTest/debug/app-debug-androidTest.apk" \
-        --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
+        --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0" \
+        --build_tag "${{ parameters.packageName }}"
+
     displayName: Run E2E tests using Browserstack
-    workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk
+    workingDirectory: $(Build.BinariesDirectory)
     timeoutInMinutes: 15
     env:
       BROWSERSTACK_ID: $(browserstack_username)

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml

@@ -94,6 +94,7 @@ jobs:
 
          # Mount qnn volume if building qnn android package
         if [[ ${{ parameters.packageName }} == *qnn* ]]; then
+          echo $(QnnSDKVersionYaml)
           QNN_VOLUME="--volume $(QnnSDKRootDir):/qnn_home"
           USE_QNN="1"
         else

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -48,6 +48,10 @@ parameters:
   type: string
   default: '0'
 
+- name: QnnSDKVersion
+  type: string
+  default: '2.26.0.240828'
+
 stages:
 - template: linux-cpu-packaging-pipeline.yml
   parameters:
@@ -88,10 +92,16 @@ stages:
       buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/android/default_qnn_aar_build_settings.json'
       artifactName: 'onnxruntime-android-qnn-aar'
       job_name_suffix: 'QNN'
-      publish_executables: '0'
+      publish_executables: '1'
       enable_code_sign: ${{ parameters.DoEsrp }}
       packageName: 'onnxruntime-android-qnn'
-  #TODO: Add test job for QNN Android AAR
+
+  - template: android-java-api-aar-test.yml
+    parameters:
+      artifactName: 'onnxruntime-android-qnn-aar'
+      job_name_suffix: 'QNN'
+      packageName: 'onnxruntime-android-qnn'
+      QnnSDKVersion: ${{ parameters.QnnSDKVersion }}
 
 - stage: iOS_Full_xcframework
   dependsOn: []

tools/ci_build/github/azure-pipelines/templates/jobs/download_linux_qnn_sdk.yml

@@ -16,6 +16,21 @@ steps:
       echo $(QnnSDKRootDir)
     displayName: 'Print QnnSDKRootDir after downloading QNN SDK'
 
+  - script: |
+      sdk_file="$(QnnSDKRootDir)/sdk.yaml"
+      qnn_sdk_version=$(grep '^version:' "$sdk_file" | head -n 1 | cut -d':' -f2 | xargs | cut -d'.' -f1-3)
+      if [[ -n "$qnn_sdk_version" ]]; then
+        echo "##vso[task.setvariable variable=QnnSDKVersionYaml]$qnn_sdk_version"
+      else
+        echo "Version not found in sdk.yaml."
+        exit 1
+      fi
+    displayName: Set QnnSDKVersionYaml based on sdk.yaml
+
+  - bash: |
+      echo "QnnSDKVersionYaml=$(QnnSDKVersionYaml)"
+    displayName: 'Print QnnSDKVersionYaml after extracting from sdk.yaml'
+
   - script: |
       azcopy cp --recursive 'https://lotusscus.blob.core.windows.net/models/qnnsdk/Qualcomm AI Hub Proprietary License.pdf' $(QnnSDKRootDir)
     displayName: 'Download Qualcomm AI Hub license'

tools/python/upload_and_run_browserstack_tests.py

@@ -29,13 +29,16 @@ def upload_apk_parse_json(post_url, apk_path, id, token):
     return response_to_json(response)
 
 
-def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token):
+def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token, project, build_tag):
     headers = {}
 
     json_data = {
         "devices": devices,
         "app": app_url,
         "testSuite": test_suite_url,
+        "project": project,
+        "buildTag": build_tag,
+        "deviceLogs": True,
     }
 
     build_response = requests.post(
@@ -106,6 +109,13 @@ def build_query_loop(build_id, test_platform, id, token):
         required=True,
     )
 
+    parser.add_argument(
+        "--project",
+        type=str,
+        help="Identifier to logically group multiple builds together",
+        default="ONNXRuntime tests",
+    )
+    parser.add_argument("--build_tag", type=str, help="Identifier to tag the build with a unique name", default="")
     args = parser.parse_args()
 
     try:
@@ -140,6 +150,8 @@ def build_query_loop(build_id, test_platform, id, token):
         args.test_platform,
         browserstack_id,
         browserstack_token,
+        args.project,
+        args.build_tag,
     )
 
     # Get build status until the tests are no longer running