1// Copyright 2024 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14package tradefed_modules 15 16import ( 17 "android/soong/android" 18 "android/soong/java" 19 "android/soong/sh" 20 "fmt" 21 "strconv" 22 "strings" 23 "testing" 24 25 "github.com/google/blueprint" 26) 27 28const bp = ` 29 android_app { 30 name: "foo", 31 srcs: ["a.java"], 32 sdk_version: "current", 33 } 34 35 android_test_helper_app { 36 name: "HelperApp", 37 srcs: ["helper.java"], 38 } 39 40 android_test { 41 name: "base", 42 sdk_version: "current", 43 data: [":HelperApp", "data/testfile"], 44 host_required: ["other-module"], 45 test_suites: ["general-tests"], 46 } 47 48 test_module_config { 49 name: "derived_test", 50 base: "base", 51 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 52 include_annotations: ["android.platform.test.annotations.LargeTest"], 53 test_suites: ["general-tests"], 54 } 55 56` 57 58const variant = "android_arm64_armv8-a" 59 60// Ensure we create files needed and set the AndroidMkEntries needed 61func TestModuleConfigAndroidTest(t *testing.T) { 62 63 ctx := android.GroupFixturePreparers( 64 java.PrepareForTestWithJavaDefaultModules, 65 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 66 ).RunTestWithBp(t, bp) 67 68 derived := ctx.ModuleForTests("derived_test", variant) 69 // Assert there are rules to create these files. 70 derived.Output("test_module_config.manifest") 71 derived.Output("test_config_fixer/derived_test.config") 72 73 // Ensure some basic rules exist. 74 ctx.ModuleForTests("base", "android_common").Output("package-res.apk") 75 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 76 77 // Ensure some entries from base are there, specifically support files for data and helper apps. 78 // Do not use LOCAL_COMPATIBILITY_SUPPORT_FILES, but instead use LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES 79 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 80 []string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk", 81 "out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk", 82 "out/soong/target/product/test_device/testcases/derived_test/data/testfile"}, 83 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 84 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{}) 85 86 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_REQUIRED_MODULES"], []string{"base"}) 87 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_HOST_REQUIRED_MODULES"], []string{"other-module"}) 88 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_CERTIFICATE"], []string{"build/make/target/product/security/testkey.x509.pem"}) 89 android.AssertStringEquals(t, "", entries.Class, "APPS") 90 91 // And some new derived entries are there. 92 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE_TAGS"], []string{"tests"}) 93 94 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], fmt.Sprintf("derived_test/%s/test_config_fixer/derived_test.config", variant)) 95 96 // Check the footer lines. Our support files should depend on base's support files. 97 convertedActual := make([]string, 5) 98 for i, e := range entries.FooterLinesForTests() { 99 // AssertStringPathsRelativeToTop doesn't replace both instances 100 convertedActual[i] = strings.Replace(e, ctx.Config.SoongOutDir(), "", 2) 101 } 102 android.AssertArrayString(t, fmt.Sprintf("%s", ctx.Config.SoongOutDir()), convertedActual, []string{ 103 "include $(BUILD_SYSTEM)/soong_app_prebuilt.mk", 104 "/target/product/test_device/testcases/derived_test/arm64/base.apk: /target/product/test_device/testcases/base/arm64/base.apk", 105 "/target/product/test_device/testcases/derived_test/HelperApp.apk: /target/product/test_device/testcases/base/HelperApp.apk", 106 "/target/product/test_device/testcases/derived_test/data/testfile: /target/product/test_device/testcases/base/data/testfile", 107 "", 108 }) 109} 110 111func TestModuleConfigShTest(t *testing.T) { 112 ctx := android.GroupFixturePreparers( 113 sh.PrepareForTestWithShBuildComponents, 114 android.PrepareForTestWithAndroidBuildComponents, 115 android.FixtureMergeMockFs(android.MockFS{ 116 "test.sh": nil, 117 "testdata/data1": nil, 118 "testdata/sub/data2": nil, 119 }), 120 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 121 ).RunTestWithBp(t, ` 122 sh_test { 123 name: "shell_test", 124 src: "test.sh", 125 filename: "test.sh", 126 test_suites: ["general-tests"], 127 data: [ 128 "testdata/data1", 129 "testdata/sub/data2", 130 ], 131 } 132 test_module_config { 133 name: "conch", 134 base: "shell_test", 135 test_suites: ["general-tests"], 136 options: [{name: "SomeName", value: "OptionValue"}], 137 } 138 `) 139 derived := ctx.ModuleForTests("conch", variant) // 140 conch := derived.Module().(*testModuleConfigModule) 141 android.AssertArrayString(t, "TestcaseRelDataFiles", []string{"arm64/testdata/data1", "arm64/testdata/sub/data2"}, conch.provider.TestcaseRelDataFiles) 142 android.AssertStringEquals(t, "Rel OutputFile", "test.sh", conch.provider.OutputFile.Rel()) 143 144 // Assert there are rules to create these files. 145 derived.Output("test_module_config.manifest") 146 derived.Output("test_config_fixer/conch.config") 147 148 // Ensure some basic rules exist. 149 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 150 151 // Ensure some entries from base are there, specifically support files for data and helper apps. 152 // Do not use LOCAL_COMPATIBILITY_SUPPORT_FILES, but instead use LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES 153 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 154 []string{"out/soong/target/product/test_device/testcases/conch/arm64/testdata/data1", 155 "out/soong/target/product/test_device/testcases/conch/arm64/testdata/sub/data2"}, 156 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 157 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{}) 158 159 android.AssertStringEquals(t, "app class", "NATIVE_TESTS", entries.Class) 160 android.AssertArrayString(t, "required modules", []string{"shell_test"}, entries.EntryMap["LOCAL_REQUIRED_MODULES"]) 161 android.AssertArrayString(t, "host required modules", []string{}, entries.EntryMap["LOCAL_HOST_REQUIRED_MODULES"]) 162 android.AssertArrayString(t, "cert", []string{}, entries.EntryMap["LOCAL_CERTIFICATE"]) 163 164 // And some new derived entries are there. 165 android.AssertArrayString(t, "tags", []string{}, entries.EntryMap["LOCAL_MODULE_TAGS"]) 166 167 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], 168 fmt.Sprintf("conch/%s/test_config_fixer/conch.config", variant)) 169 170 // Check the footer lines. Our support files should depend on base's support files. 171 convertedActual := make([]string, 4) 172 for i, e := range entries.FooterLinesForTests() { 173 // AssertStringPathsRelativeToTop doesn't replace both instances 174 convertedActual[i] = strings.Replace(e, ctx.Config.SoongOutDir(), "", 2) 175 } 176 android.AssertArrayString(t, fmt.Sprintf("%s", ctx.Config.SoongOutDir()), convertedActual, []string{ 177 "include $(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk", 178 "/target/product/test_device/testcases/conch/arm64/testdata/data1: /target/product/test_device/testcases/shell_test/arm64/testdata/data1", 179 "/target/product/test_device/testcases/conch/arm64/testdata/sub/data2: /target/product/test_device/testcases/shell_test/arm64/testdata/sub/data2", 180 "", 181 }) 182 183} 184 185// Make sure we call test-config-fixer with the right args. 186func TestModuleConfigOptions(t *testing.T) { 187 188 ctx := android.GroupFixturePreparers( 189 java.PrepareForTestWithJavaDefaultModules, 190 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 191 ).RunTestWithBp(t, bp) 192 193 // Check that we generate a rule to make a new AndroidTest.xml/Module.config file. 194 derived := ctx.ModuleForTests("derived_test", variant) 195 rule_cmd := derived.Rule("fix_test_config").RuleParams.Command 196 android.AssertStringDoesContain(t, "Bad FixConfig rule inputs", rule_cmd, 197 `--test-runner-options='[{"Name":"exclude-filter","Key":"","Value":"android.test.example.devcodelab.DevCodelabTest#testHelloFail"},{"Name":"include-annotation","Key":"","Value":"android.platform.test.annotations.LargeTest"}]'`) 198} 199 200// Ensure we error for a base we don't support. 201func TestModuleConfigWithHostBaseShouldFailWithExplicitMessage(t *testing.T) { 202 badBp := ` 203 java_test { 204 name: "base", 205 srcs: ["a.java"], 206 } 207 208 test_module_config { 209 name: "derived_test", 210 base: "base", 211 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 212 include_annotations: ["android.platform.test.annotations.LargeTest"], 213 test_suites: ["general-tests"], 214 }` 215 216 android.GroupFixturePreparers( 217 java.PrepareForTestWithJavaDefaultModules, 218 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 219 ).ExtendWithErrorHandler( 220 android.FixtureExpectsAtLeastOneErrorMatchingPattern("'base' module used as base but it is not a 'android_test' module.")). 221 RunTestWithBp(t, badBp) 222} 223 224func TestModuleConfigBadBaseShouldFailWithGeneralMessage(t *testing.T) { 225 badBp := ` 226 java_library { 227 name: "base", 228 srcs: ["a.java"], 229 } 230 231 test_module_config { 232 name: "derived_test", 233 base: "base", 234 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 235 include_annotations: ["android.platform.test.annotations.LargeTest"], 236 test_suites: ["general-tests"], 237 }` 238 239 android.GroupFixturePreparers( 240 java.PrepareForTestWithJavaDefaultModules, 241 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 242 ).ExtendWithErrorHandler( 243 android.FixtureExpectsAtLeastOneErrorMatchingPattern("'base' module used as base but it is not a 'android_test' module.")). 244 RunTestWithBp(t, badBp) 245} 246 247func TestModuleConfigNoBaseShouldFail(t *testing.T) { 248 badBp := ` 249 java_library { 250 name: "base", 251 srcs: ["a.java"], 252 } 253 254 test_module_config { 255 name: "derived_test", 256 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 257 include_annotations: ["android.platform.test.annotations.LargeTest"], 258 test_suites: ["general-tests"], 259 }` 260 261 android.GroupFixturePreparers( 262 java.PrepareForTestWithJavaDefaultModules, 263 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 264 ).ExtendWithErrorHandler( 265 android.FixtureExpectsOneErrorPattern("'base' field must be set to a 'android_test' module.")). 266 RunTestWithBp(t, badBp) 267} 268 269// Ensure we error for a base we don't support. 270func TestModuleConfigNoFiltersOrAnnotationsShouldFail(t *testing.T) { 271 badBp := ` 272 android_test { 273 name: "base", 274 sdk_version: "current", 275 srcs: ["a.java"], 276 test_suites: ["general-tests"], 277 } 278 279 test_module_config { 280 name: "derived_test", 281 base: "base", 282 test_suites: ["general-tests"], 283 }` 284 285 ctx := android.GroupFixturePreparers( 286 java.PrepareForTestWithJavaDefaultModules, 287 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 288 ).ExtendWithErrorHandler( 289 android.FixtureExpectsAtLeastOneErrorMatchingPattern("Test options must be given")). 290 RunTestWithBp(t, badBp) 291 ctx.ModuleForTests("derived_test", variant) 292} 293 294func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) { 295 multiBp := ` 296 android_test { 297 name: "base", 298 sdk_version: "current", 299 srcs: ["a.java"], 300 data: [":HelperApp", "data/testfile"], 301 test_suites: ["general-tests"], 302 } 303 304 android_test_helper_app { 305 name: "HelperApp", 306 srcs: ["helper.java"], 307 } 308 309 test_module_config { 310 name: "derived_test", 311 base: "base", 312 include_annotations: ["android.platform.test.annotations.LargeTest"], 313 test_suites: ["general-tests"], 314 } 315 316 test_module_config { 317 name: "another_derived_test", 318 base: "base", 319 include_annotations: ["android.platform.test.annotations.LargeTest"], 320 test_suites: ["general-tests"], 321 }` 322 323 ctx := android.GroupFixturePreparers( 324 java.PrepareForTestWithJavaDefaultModules, 325 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 326 ).RunTestWithBp(t, multiBp) 327 328 { 329 derived := ctx.ModuleForTests("derived_test", variant) 330 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 331 // All these should be the same in both derived tests 332 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 333 []string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk", 334 "out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk", 335 "out/soong/target/product/test_device/testcases/derived_test/data/testfile"}, 336 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 337 338 // Except this one, which points to the updated tradefed xml file. 339 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], fmt.Sprintf("derived_test/%s/test_config_fixer/derived_test.config", variant)) 340 // And this one, the module name. 341 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"}) 342 } 343 344 { 345 derived := ctx.ModuleForTests("another_derived_test", variant) 346 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 347 // All these should be the same in both derived tests 348 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 349 []string{"out/soong/target/product/test_device/testcases/another_derived_test/arm64/base.apk", 350 "out/soong/target/product/test_device/testcases/another_derived_test/HelperApp.apk", 351 "out/soong/target/product/test_device/testcases/another_derived_test/data/testfile"}, 352 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 353 // Except this one, which points to the updated tradefed xml file. 354 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], 355 fmt.Sprintf("another_derived_test/%s/test_config_fixer/another_derived_test.config", variant)) 356 // And this one, the module name. 357 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"another_derived_test"}) 358 } 359} 360 361// Test_module_config_host rule is allowed to depend on java_test_host 362func TestModuleConfigHostBasics(t *testing.T) { 363 bp := ` 364 java_test_host { 365 name: "base", 366 srcs: ["a.java"], 367 test_suites: ["suiteA", "general-tests", "suiteB"], 368 } 369 370 test_module_config_host { 371 name: "derived_test", 372 base: "base", 373 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 374 include_annotations: ["android.platform.test.annotations.LargeTest"], 375 test_suites: ["general-tests"], 376 }` 377 378 ctx := android.GroupFixturePreparers( 379 java.PrepareForTestWithJavaDefaultModules, 380 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 381 ).RunTestWithBp(t, bp) 382 383 variant := ctx.Config.BuildOS.String() + "_common" 384 derived := ctx.ModuleForTests("derived_test", variant) 385 mod := derived.Module().(*testModuleConfigHostModule) 386 allEntries := android.AndroidMkEntriesForTest(t, ctx.TestContext, mod) 387 entries := allEntries[0] 388 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"}) 389 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SDK_VERSION"], []string{"private_current"}) 390 android.AssertStringEquals(t, "", entries.Class, "JAVA_LIBRARIES") 391 392 if !mod.Host() { 393 t.Errorf("host bit is not set for a java_test_host module.") 394 } 395 actualData, _ := strconv.ParseBool(entries.EntryMap["LOCAL_IS_UNIT_TEST"][0]) 396 android.AssertBoolEquals(t, "LOCAL_IS_UNIT_TEST", true, actualData) 397 398} 399 400// When you pass an 'android_test' as base, the warning message is a bit obscure, 401// talking about variants, but it is something. Ideally we could do better. 402func TestModuleConfigHostBadBaseShouldFailWithVariantWarning(t *testing.T) { 403 badBp := ` 404 android_test { 405 name: "base", 406 sdk_version: "current", 407 srcs: ["a.java"], 408 } 409 410 test_module_config_host { 411 name: "derived_test", 412 base: "base", 413 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 414 include_annotations: ["android.platform.test.annotations.LargeTest"], 415 }` 416 417 android.GroupFixturePreparers( 418 java.PrepareForTestWithJavaDefaultModules, 419 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 420 ).ExtendWithErrorHandler( 421 android.FixtureExpectsAtLeastOneErrorMatchingPattern("missing variant")). 422 RunTestWithBp(t, badBp) 423} 424 425func TestModuleConfigHostNeedsATestSuite(t *testing.T) { 426 badBp := ` 427 java_test_host { 428 name: "base", 429 srcs: ["a.java"], 430 } 431 432 test_module_config_host { 433 name: "derived_test", 434 base: "base", 435 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 436 include_annotations: ["android.platform.test.annotations.LargeTest"], 437 }` 438 439 android.GroupFixturePreparers( 440 java.PrepareForTestWithJavaDefaultModules, 441 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 442 ).ExtendWithErrorHandler( 443 android.FixtureExpectsAtLeastOneErrorMatchingPattern("At least one test-suite must be set")). 444 RunTestWithBp(t, badBp) 445} 446 447func TestModuleConfigNonMatchingTestSuitesGiveErrors(t *testing.T) { 448 badBp := ` 449 java_test_host { 450 name: "base", 451 srcs: ["a.java"], 452 test_suites: ["general-tests", "some-compat"], 453 } 454 455 test_module_config_host { 456 name: "derived_test", 457 base: "base", 458 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 459 include_annotations: ["android.platform.test.annotations.LargeTest"], 460 test_suites: ["device-tests", "random-suite"], 461 }` 462 463 android.GroupFixturePreparers( 464 java.PrepareForTestWithJavaDefaultModules, 465 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 466 ).ExtendWithErrorHandler( 467 // Use \\ to escape bracket so it isn't used as [] set for regex. 468 android.FixtureExpectsAtLeastOneErrorMatchingPattern("Suites: \\[device-tests, random-suite] listed but do not exist in base module")). 469 RunTestWithBp(t, badBp) 470} 471 472func TestTestOnlyProvider(t *testing.T) { 473 t.Parallel() 474 ctx := android.GroupFixturePreparers( 475 java.PrepareForTestWithJavaDefaultModules, 476 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 477 ).RunTestWithBp(t, ` 478 // These should be test-only 479 test_module_config_host { 480 name: "host-derived-test", 481 base: "host-base", 482 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 483 include_annotations: ["android.platform.test.annotations.LargeTest"], 484 test_suites: ["general-tests"], 485 } 486 487 test_module_config { 488 name: "derived-test", 489 base: "base", 490 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 491 include_annotations: ["android.platform.test.annotations.LargeTest"], 492 test_suites: ["general-tests"], 493 } 494 495 android_test { 496 name: "base", 497 sdk_version: "current", 498 data: ["data/testfile"], 499 test_suites: ["general-tests"], 500 } 501 502 java_test_host { 503 name: "host-base", 504 srcs: ["a.java"], 505 test_suites: ["general-tests"], 506 }`, 507 ) 508 509 // Visit all modules and ensure only the ones that should 510 // marked as test-only are marked as test-only. 511 512 actualTestOnly := []string{} 513 ctx.VisitAllModules(func(m blueprint.Module) { 514 if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { 515 if provider.TestOnly { 516 actualTestOnly = append(actualTestOnly, m.Name()) 517 } 518 } 519 }) 520 expectedTestOnlyModules := []string{ 521 "host-derived-test", 522 "derived-test", 523 // android_test and java_test_host are tests too. 524 "host-base", 525 "base", 526 } 527 528 notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) 529 if notEqual { 530 t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) 531 } 532} 533