1#!/bin/bash
2#
3# Build benchmark app and run it, mimicking a user-initiated run
4#
5# Output is logged to a temporary folder and summarized in txt and JSON formats.
6# parallel-inference-stress tests produce no output except for the success or failure notification,
7# which is not logged.
8
9if [[ "$OSTYPE" == "darwin"* ]]; then
10  OPTS="$(getopt f:rbsm:xn: -- "$*")"
11else
12  OPTS="$(getopt -o f:rbsm:xn: -l filter-driver:,include-nnapi-reference,nnapi-reference-only,skip-build,use-nnapi-sl,filter-model,extract-nnapi-sl,nnapi-sl-vendor -- "$@")"
13fi
14
15if [ $? -ne 0 ]; then
16  echo "Invalid arguments, accepted options are"
17  if [[ "$OSTYPE" == "darwin"* ]]; then
18    echo " -f <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression"
19    echo " -r : to include nnapi-reference in target drivers"
20    echo " -b : skip build and installation of tests"
21    echo " -s : use NNAPI Support Library drivers (embedded in the benchmark APK unless -x is specified)"
22    echo " -x : extract NNAPI Support Library drivers from the APK"
23    echo " -n <nnapi-sl-vendor> : Optional, specify which SL driver vendor is used (qc/mtk/arm), default: qc"
24    echo " -m <regex> : to filter the models used in the tests"
25  else
26    echo " -f <regex> | --filter-driver <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression"
27    echo " -r | --include-nnapi-reference : to include nnapi-reference in target drivers"
28    echo " --nnapi-reference-only : to run tests only vs nnapi-reference"
29    echo " -b | --skip-build : skip build and installation of tests"
30    echo " -s | --use-nnapi-sl : use NNAPI Support Library drivers (embedded in the benchmark APK unless -x is specified)"
31    echo " -x | --extract-nnapi-sl : extract NNAPI Support Library drivers from the APK"
32    echo " -n <nnapi-sl-vendor> : Optional, specify which SL driver vendor is used (qc/mtk/arm), default: qc"
33    echo " -m <regex> : to filter the models used in the tests"
34  fi
35  exit
36fi
37
38eval set -- "$OPTS"
39
40DRIVER_FILTER_OPT=""
41INCLUDE_NNAPI_REF_OPT=""
42BUILD_AND_INSTALL=true
43NNAPI_SL_FILTER_OPT=""
44MODEL_FILTER_OPT=""
45while [ $# -gt 0 ] ; do
46  case "$1" in
47    -f|--filter-driver)
48      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter $2"
49      shift 2
50      ;;
51    -r|--include-nnapi-reference)
52      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
53      shift
54      ;;
55    --nnapi-reference-only)
56      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter no-device"
57      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
58      shift
59      ;;
60    -m|--filter-model)
61      MODEL_FILTER_OPT="-e nnBenchmarkModelFilter $2"
62      shift 2
63      ;;
64    -b|--skip-build)
65      BUILD_AND_INSTALL=false
66      shift
67      ;;
68    -s|--use-nnapi-sl)
69      NNAPI_SL_FILTER_OPT+=" -e useNnApiSupportLibrary true"
70      shift
71      ;;
72    -x|--extract-nnapi-sl)
73      NNAPI_SL_FILTER_OPT+=" -e extractNnApiSupportLibrary true"
74      shift
75
76      echo "Creating configuration file with list of libraries"
77      mkdir sl_prebuilt/assets
78      ls sl_prebuilt/ 2>/dev/null | grep '.so'  >sl_prebuilt/assets/sl_prebuilt_filelist.txt
79      ;;
80    -n|--nnapi-sl-vendor)
81      NNAPI_SL_VENDOR=$2
82      shift 2
83      ;;
84    --)
85      shift
86      break
87      ;;
88    *)
89      echo "Unsupported arg $1"
90      exit 1
91  esac
92done
93
94MODE="${1:-scoring}"
95INSTALL_NATIVE_TESTS=false
96CRASH_TEST_APP="NeuralNetworksApiCrashTest"
97APP="NeuralNetworksApiBenchmark"
98case "$MODE" in
99  scoring)
100    CLASS=com.android.nn.benchmark.app.NNScoringTest
101    ;;
102  inference-stress)
103    CLASS=com.android.nn.benchmark.app.NNInferenceStressTest
104    ;;
105  model-loading-stress)
106    CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest
107    ;;
108  parallel-inference-stress)
109    CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest
110    APP="$CRASH_TEST_APP"
111    ;;
112  parallel-inference-stress-in-process)
113    CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest
114    APP="$CRASH_TEST_APP"
115    ;;
116  client-early-termination-stress)
117    CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest
118    APP="$CRASH_TEST_APP"
119    ;;
120  multi-process-inference-stress)
121    CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest
122    APP="$CRASH_TEST_APP"
123    INSTALL_NATIVE_TESTS=true
124    ;;
125  multi-process-model-load-stress)
126    CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest
127    APP="$CRASH_TEST_APP"
128    INSTALL_NATIVE_TESTS=true
129    ;;
130  memory-mapped-model-load-stress)
131    CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest
132    APP="$CRASH_TEST_APP"
133    ;;
134  model-load-random-stress)
135    APP="$CRASH_TEST_APP"
136    CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest
137    ;;
138  inference-random-stress)
139    APP="$CRASH_TEST_APP"
140    CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest
141    ;;
142  performance-degradation-stress)
143    APP="$CRASH_TEST_APP"
144    CLASS=com.android.nn.crashtest.app.NNPerformanceDegradationTest
145    ;;
146  *)
147    echo "Unknown execution mode: $1"
148    echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \
149      "parallel-inference-stress, parallel-inference-stress-in-process, " \
150      "client-early-termination-stress, multi-process-inference-stress, " \
151      "multi-process-model-load-stress memory-mapped-model-load-stress, " \
152      "model-load-random-stress, inference-random-stress, performance-degradation-stress"
153    exit 1
154    ;;
155esac
156
157if [[ -z "$ANDROID_BUILD_TOP" ]]; then
158  echo ANDROID_BUILD_TOP not set, bailing out
159  echo you must run lunch before running this script
160  exit 1
161fi
162
163set -e
164cd $ANDROID_BUILD_TOP
165
166if [ "$BUILD_AND_INSTALL" = true ]; then
167   if [ ! -z "$NNAPI_SL_FILTER_OPT" ]; then
168    SL_PREBUILT=test/mlts/benchmark/sl_prebuilt
169    if [ ! -n "$(ls -A $SL_PREBUILT/*.so 2>/dev/null)" ]; then
170      echo "There is no NNAPI SL binary file under $ANDROID_BUILD_TOP/$SL_PREBUILT, cannot test using NNAPI SL"
171      exit
172    fi
173    if [ ! -f "$SL_PREBUILT/Android.bp" ]; then
174      echo "================================================================"
175      echo "Enabling build of NNAPI SL libraries using template definition."
176      echo  "If the definitions in $SL_PREBUILT/Android.bp don't match the libraries you copied"
177      echo " please define your own version of $SL_PREBUILT/Android.bp"
178      echo "================================================================"
179      mv $SL_PREBUILT/Android.bp.template $SL_PREBUILT/Android.bp
180    fi
181    if [[ -z "$NNAPI_SL_VENDOR" ]]; then
182      NNAPI_SL_VENDOR="qc"
183    fi
184    NNAPI_SL_FILTER_OPT+=" -e nnApiSupportLibraryVendor $NNAPI_SL_VENDOR"
185  fi
186
187  # Build and install benchmark app
188  TMPFILE=$(mktemp)
189  build/soong/soong_ui.bash --make-mode ${APP} 2>&1 | tee ${TMPFILE}
190  TARGET_ARCH=$(cat ${TMPFILE} | grep TARGET_ARCH= | sed -e 's/TARGET_ARCH=//')
191  if [ "${TARGET_ARCH}" = "aarch64" ]; then
192      APK_DIR=arm64
193  else
194      APK_DIR=${TARGET_ARCH}
195  fi
196
197  if [ ! -z "$NNAPI_SL_FILTER_OPT" ]; then
198    if [ "$(unzip -l $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk | grep libnnapi_sl_driver | wc -l)" -ne 1 ]; then
199      echo "NNAPI SL Libraries are not included in the APK" \
200          "please check the library list is included in the LOCAL_JNI_SHARED_LIBRARIES list " \
201          "for ${APP}. Please check the value of SL_LIBS in Android.mk"
202      exit
203    fi
204  fi
205
206  if ! adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk; then
207    adb uninstall com.android.nn.benchmark.app
208    adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk
209  fi
210
211  if [ "$INSTALL_NATIVE_TESTS" = true ]; then
212    build/soong/soong_ui.bash --make-mode nn_stress_test
213    adb push $OUT/system/bin/nn_stress_test /bin/
214  fi
215fi
216
217# Should we figure out if we run on release device
218if [ -z "$MLTS_RELEASE_DEVICE" ]; then
219  BUILD_DESCRIPTION=`adb shell getprop ro.build.description`
220  if [[ $BUILD_DESCRIPTION =~ .*release.* ]]
221  then
222    MLTS_RELEASE_DEVICE=True
223  else
224    MLTS_RELEASE_DEVICE=False
225  fi
226fi
227
228# Pass --no-isolated-storage to am instrument?
229BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release`
230AM_INSTRUMENT_FLAGS="$DRIVER_FILTER_OPT $INCLUDE_NNAPI_REF_OPT $NNAPI_SL_FILTER_OPT $MODEL_FILTER_OPT"
231if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then
232  AM_INSTRUMENT_FLAGS+=" --no-isolated-storage"
233fi
234
235if [[ "$MODE" == "scoring" ]]; then
236  if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then
237    TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity"
238    while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do
239       echo "************************************************************"
240       echo "Grant External storage write permissions to MLTS to proceed!"
241       echo "************************************************************"
242       read -n 1 -r -p "Continue? (press any key)"
243       echo
244    done
245  else
246    adb root
247    adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE"
248    # Skip setup wizard and remount (read-write)
249    if ! adb shell test -f /data/local.prop; then
250      adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop'
251      adb shell 'chmod 644 /data/local.prop'
252      adb shell 'settings put global device_provisioned 1*'
253      adb shell 'settings put secure user_setup_complete 1'
254      adb disable-verity
255      adb reboot
256      sleep 5
257      adb wait-for-usb-device root
258      adb wait-for-usb-device remount
259      sleep 5
260    fi
261    set +e
262    # Enable menu key press through adb
263    adb shell 'echo testing > /data/local/enable_menu_key'
264    # Leave screen on (affects scheduling)
265    adb shell settings put system screen_off_timeout 86400000
266    # Stop background apps, seem to take ~10% CPU otherwise
267    adb shell 'pm disable com.google.android.googlequicksearchbox'
268    adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done;
269    set -e
270  fi
271fi
272
273adb shell setprop debug.nn.cpuonly 0
274adb shell setprop debug.nn.vlog "''"
275
276# Menukey - make sure screen is on
277adb shell "input keyevent 82"
278# Show homescreen
279adb shell wm dismiss-keyguard
280
281if [[ "$MODE" == "scoring" ]]; then
282  LOGDIR=$(mktemp -d)/mlts-logs
283  HOST_CSV=$LOGDIR/benchmark.csv
284  RESULT_HTML=$LOGDIR/result.html
285  DEVICE_CSV=/storage/emulated/0/mlts_benchmark.csv
286
287  mkdir -p $LOGDIR
288  echo Creating logs in $LOGDIR
289
290  # Remove old benchmark csv data
291  adb shell rm -f ${DEVICE_CSV}
292fi
293
294# Set the shell pid as a top-app and run tests
295time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner"
296
297if [[ "$MODE" == "scoring" ]]; then
298  adb pull $DEVICE_CSV $HOST_CSV
299  echo Benchmark data saved in $HOST_CSV
300
301  $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML
302  echo Results stored  in $RESULT_HTML
303  xdg-open $RESULT_HTML
304fi
305