1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18#include "interpreter/cfi_asm_support.h" 19 20#include "arch/quick_alloc_entrypoints.S" 21#include "arch/quick_field_entrypoints.S" 22 23.macro SAVE_REG_INCREASE_FRAME reg, frame_adjustment 24 str \reg, [sp, #-(\frame_adjustment)]! 25 .cfi_adjust_cfa_offset (\frame_adjustment) 26 .cfi_rel_offset \reg, 0 27.endm 28 29.macro RESTORE_REG_DECREASE_FRAME reg, frame_adjustment 30 ldr \reg, [sp], #(\frame_adjustment) 31 .cfi_restore \reg 32 .cfi_adjust_cfa_offset -(\frame_adjustment) 33.endm 34 35.macro POP_SAVE_REFS_ONLY_FRAME 36 DECREASE_FRAME 96 37.endm 38 39 /* 40 * Macro that sets up the callee save frame to conform with 41 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). 42 * 43 * TODO This is probably too conservative - saving FP & LR. 44 */ 45.macro SETUP_SAVE_REFS_AND_ARGS_FRAME 46 // art::Runtime* xIP0 = art::Runtime::instance_; 47 // Our registers aren't intermixed - just spill in order. 48 LOAD_RUNTIME_INSTANCE xIP0 49 50 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefAndArgs]; 51 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 52 53 INCREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 54 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 55 56 str xIP0, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsAndArgs]. 57 // Place sp in Thread::Current()->top_quick_frame. 58 mov xIP0, sp 59 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 60.endm 61 62.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 63 INCREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 64 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 65 str x0, [sp, #0] // Store ArtMethod* to bottom of stack. 66 // Place sp in Thread::Current()->top_quick_frame. 67 mov xIP0, sp 68 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 69.endm 70 71 /* 72 * Macro that sets up the callee save frame to conform with 73 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 74 * when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING 75 * and saving registers x29 and LR is handled elsewhere. 76 */ 77.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \ 78 runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 79 // Ugly compile-time check, but we only have the preprocessor. 80#if (FRAME_SIZE_SAVE_EVERYTHING != 512) 81#error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected." 82#endif 83 84 // Save FP registers. 85 stp d0, d1, [sp, #16] 86 stp d2, d3, [sp, #32] 87 stp d4, d5, [sp, #48] 88 stp d6, d7, [sp, #64] 89 stp d8, d9, [sp, #80] 90 stp d10, d11, [sp, #96] 91 stp d12, d13, [sp, #112] 92 stp d14, d15, [sp, #128] 93 stp d16, d17, [sp, #144] 94 stp d18, d19, [sp, #160] 95 stp d20, d21, [sp, #176] 96 stp d22, d23, [sp, #192] 97 stp d24, d25, [sp, #208] 98 stp d26, d27, [sp, #224] 99 stp d28, d29, [sp, #240] 100 stp d30, d31, [sp, #256] 101 102 // Save core registers. 103 SAVE_TWO_REGS x0, x1, 272 104 SAVE_TWO_REGS x2, x3, 288 105 SAVE_TWO_REGS x4, x5, 304 106 SAVE_TWO_REGS x6, x7, 320 107 SAVE_TWO_REGS x8, x9, 336 108 SAVE_TWO_REGS x10, x11, 352 109 SAVE_TWO_REGS x12, x13, 368 110 SAVE_TWO_REGS x14, x15, 384 111 SAVE_TWO_REGS x16, x17, 400 // Do not save the platform register. 112 SAVE_TWO_REGS x19, x20, 416 113 SAVE_TWO_REGS x21, x22, 432 114 SAVE_TWO_REGS x23, x24, 448 115 SAVE_TWO_REGS x25, x26, 464 116 SAVE_TWO_REGS x27, x28, 480 117 118 // art::Runtime* xIP0 = art::Runtime::instance_; 119 LOAD_RUNTIME_INSTANCE xIP0 120 121 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything]; 122 ldr xIP0, [xIP0, \runtime_method_offset] 123 124 // Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything]. 125 str xIP0, [sp] 126 // Place sp in Thread::Current()->top_quick_frame. 127 mov xIP0, sp 128 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 129.endm 130 131 /* 132 * Macro that sets up the callee save frame to conform with 133 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 134 */ 135.macro SETUP_SAVE_EVERYTHING_FRAME runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 136 INCREASE_FRAME 512 137 SAVE_TWO_REGS x29, xLR, 496 138 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \runtime_method_offset 139.endm 140 141.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 142 // Restore FP registers. 143 ldp d0, d1, [sp, #16] 144 ldp d2, d3, [sp, #32] 145 ldp d4, d5, [sp, #48] 146 ldp d6, d7, [sp, #64] 147 ldp d8, d9, [sp, #80] 148 ldp d10, d11, [sp, #96] 149 ldp d12, d13, [sp, #112] 150 ldp d14, d15, [sp, #128] 151 ldp d16, d17, [sp, #144] 152 ldp d18, d19, [sp, #160] 153 ldp d20, d21, [sp, #176] 154 ldp d22, d23, [sp, #192] 155 ldp d24, d25, [sp, #208] 156 ldp d26, d27, [sp, #224] 157 ldp d28, d29, [sp, #240] 158 ldp d30, d31, [sp, #256] 159 160 // Restore core registers, except x0. 161 RESTORE_REG x1, 280 162 RESTORE_TWO_REGS x2, x3, 288 163 RESTORE_TWO_REGS x4, x5, 304 164 RESTORE_TWO_REGS x6, x7, 320 165 RESTORE_TWO_REGS x8, x9, 336 166 RESTORE_TWO_REGS x10, x11, 352 167 RESTORE_TWO_REGS x12, x13, 368 168 RESTORE_TWO_REGS x14, x15, 384 169 RESTORE_TWO_REGS x16, x17, 400 // Do not restore the platform register. 170 RESTORE_TWO_REGS x19, x20, 416 171 RESTORE_TWO_REGS x21, x22, 432 172 RESTORE_TWO_REGS x23, x24, 448 173 RESTORE_TWO_REGS x25, x26, 464 174 RESTORE_TWO_REGS x27, x28, 480 175 RESTORE_TWO_REGS x29, xLR, 496 176 177 DECREASE_FRAME 512 178.endm 179 180.macro RESTORE_SAVE_EVERYTHING_FRAME 181 RESTORE_REG x0, 272 182 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 183.endm 184 185.macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION is_ref = 0 186 ldr x1, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 187 CFI_REMEMBER_STATE 188 cbnz x1, 1f 189 DEOPT_OR_RETURN x1, \is_ref // Check if deopt is required 1901: // deliver exception on current thread 191 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 192 DELIVER_PENDING_EXCEPTION 193.endm 194 195.macro RETURN_REF_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 196 RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION /* is_ref= */ 1 197.endm 198 199.macro DEOPT_OR_RETURN temp, is_ref = 0 200 ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 201 cbnz \temp, 2f 202 ret 2032: 204 SETUP_SAVE_EVERYTHING_FRAME 205 mov x2, \is_ref // pass if result is a reference 206 mov x1, x0 // pass the result 207 mov x0, xSELF // Thread::Current 208 bl artDeoptimizeIfNeeded 209 210 CFI_REMEMBER_STATE 211 cbnz x0, 3f 212 213 RESTORE_SAVE_EVERYTHING_FRAME 214 REFRESH_MARKING_REGISTER 215 ret 216 2173: 218 // Deoptimize. 219 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 220 bl art_quick_do_long_jump // (Context*) 221 brk 0 // Unreached 222.endm 223 224.macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 temp, is_ref 225 ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 226 CFI_REMEMBER_STATE 227 cbnz \temp, 2f 228 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 229 REFRESH_MARKING_REGISTER 230 ret 2312: 232 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 233 str x0, [sp, #SAVE_EVERYTHING_FRAME_X0_OFFSET] // update result in the frame 234 mov x2, \is_ref // pass if result is a reference 235 mov x1, x0 // pass the result 236 mov x0, xSELF // Thread::Current 237 bl artDeoptimizeIfNeeded 238 239 CFI_REMEMBER_STATE 240 cbnz x0, 3f 241 242 RESTORE_SAVE_EVERYTHING_FRAME 243 REFRESH_MARKING_REGISTER 244 ret 245 2463: 247 // Deoptimize. 248 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 249 bl art_quick_do_long_jump // (Context*) 250 brk 0 // Unreached 251.endm 252 253 254.macro RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 255 CFI_REMEMBER_STATE 256 cbnz w0, 1f // result non-zero branch over 257 DEOPT_OR_RETURN x1 2581: 259 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 260 DELIVER_PENDING_EXCEPTION 261.endm 262 263.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 264 .extern \cxx_name 265ENTRY \c_name 266 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 267 mov x0, xSELF // pass Thread::Current 268 bl \cxx_name // \cxx_name(Thread*) 269 bl art_quick_do_long_jump // (Context*) 270 brk 0 // Unreached 271END \c_name 272.endm 273 274.macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 275 .extern \cxx_name 276ENTRY \c_name 277 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 278 mov x0, xSELF // pass Thread::Current 279 bl \cxx_name // \cxx_name(Thread*) 280 bl art_quick_do_long_jump // (Context*) 281 brk 0 // Unreached 282END \c_name 283.endm 284 285.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 286 .extern \cxx_name 287ENTRY \c_name 288 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context. 289 mov x1, xSELF // pass Thread::Current. 290 bl \cxx_name // \cxx_name(arg, Thread*). 291 bl art_quick_do_long_jump // (Context*) 292 brk 0 // Unreached 293END \c_name 294.endm 295 296.macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 297 .extern \cxx_name 298ENTRY \c_name 299 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 300 mov x2, xSELF // pass Thread::Current 301 bl \cxx_name // \cxx_name(arg1, arg2, Thread*) 302 bl art_quick_do_long_jump // (Context*) 303 brk 0 // Unreached 304END \c_name 305.endm 306 307 /* 308 * Called by managed code, saves callee saves and then calls artThrowException 309 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 310 */ 311ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 312 313 /* 314 * Called by managed code to create and deliver a NullPointerException. 315 */ 316NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ 317 art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 318 319 /* 320 * Call installed by a signal handler to create and deliver a NullPointerException. 321 */ 322 .extern art_quick_throw_null_pointer_exception_from_signal 323ENTRY art_quick_throw_null_pointer_exception_from_signal 324 // The fault handler pushes the gc map address, i.e. "return address", to stack 325 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 326 .cfi_def_cfa_offset __SIZEOF_POINTER__ 327 .cfi_rel_offset lr, 0 328 // Save all registers as basis for long jump context. 329 INCREASE_FRAME (FRAME_SIZE_SAVE_EVERYTHING - __SIZEOF_POINTER__) 330 SAVE_REG x29, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__) // LR already saved. 331 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 332 mov x0, lr // pass the fault address stored in LR by the fault handler. 333 mov x1, xSELF // pass Thread::Current. 334 bl artThrowNullPointerExceptionFromSignal // (arg, Thread*). 335 bl art_quick_do_long_jump // (Context*) 336 brk 0 // Unreached 337END art_quick_throw_null_pointer_exception_from_signal 338 339 /* 340 * Called by managed code to create and deliver an ArithmeticException. 341 */ 342NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 343 344 /* 345 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 346 * index, arg2 holds limit. 347 */ 348TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 349 350 /* 351 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 352 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 353 */ 354TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ 355 art_quick_throw_string_bounds, artThrowStringBoundsFromCode 356 357 /* 358 * Called by managed code to create and deliver a StackOverflowError. 359 */ 360NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 361 362 /* 363 * All generated callsites for interface invokes and invocation slow paths will load arguments 364 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 365 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 366 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 367 * 368 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 369 * of the target Method* in x0 and method->code_ in x1. 370 * 371 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 372 * thread and we branch to another stub to deliver it. 373 * 374 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 375 * pointing back to the original caller. 376 * 377 * Adapted from ARM32 code. 378 * 379 * Clobbers xIP0. 380 */ 381.macro INVOKE_TRAMPOLINE_BODY cxx_name 382 .extern \cxx_name 383 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC 384 // Helper signature is always 385 // (method_idx, *this_object, *caller_method, *self, sp) 386 387 mov x2, xSELF // pass Thread::Current 388 mov x3, sp 389 bl \cxx_name // (method_idx, this, Thread*, SP) 390 mov xIP0, x1 // save Method*->code_ 391 RESTORE_SAVE_REFS_AND_ARGS_FRAME 392 REFRESH_MARKING_REGISTER 393 cbz x0, 1f // did we find the target? if not go to exception delivery 394 br xIP0 // tail call to target 3951: 396 DELIVER_PENDING_EXCEPTION 397.endm 398.macro INVOKE_TRAMPOLINE c_name, cxx_name 399ENTRY \c_name 400 INVOKE_TRAMPOLINE_BODY \cxx_name 401END \c_name 402.endm 403 404INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, \ 405 artInvokeInterfaceTrampolineWithAccessCheck 406 407INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, \ 408 artInvokeStaticTrampolineWithAccessCheck 409INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, \ 410 artInvokeDirectTrampolineWithAccessCheck 411INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, \ 412 artInvokeSuperTrampolineWithAccessCheck 413INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, \ 414 artInvokeVirtualTrampolineWithAccessCheck 415 416 417.macro INVOKE_STUB_CREATE_FRAME 418SAVE_SIZE=8*8 // x4, x5, <padding>, x19, x20, x21, FP, LR saved. 419 SAVE_TWO_REGS_INCREASE_FRAME x4, x5, SAVE_SIZE 420 SAVE_REG x19, 24 421 SAVE_TWO_REGS x20, x21, 32 422 SAVE_TWO_REGS xFP, xLR, 48 423 424 mov xFP, sp // Use xFP for frame pointer, as it's callee-saved. 425 .cfi_def_cfa_register xFP 426 427 add x10, x2, #(__SIZEOF_POINTER__ + 0xf) // Reserve space for ArtMethod*, arguments and 428 and x10, x10, # ~0xf // round up for 16-byte stack alignment. 429 sub sp, sp, x10 // Adjust SP for ArtMethod*, args and alignment padding. 430 431 mov xSELF, x3 // Move thread pointer into SELF register. 432 433 // Copy arguments into stack frame. 434 // Use simple copy routine for now. 435 // 4 bytes per slot. 436 // X1 - source address 437 // W2 - args length 438 // X9 - destination address. 439 // W10 - temporary 440 add x9, sp, #8 // Destination address is bottom of stack + null. 441 442 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler 443 // does not have unique-id variables. 444 cbz w2, 2f 4451: 446 sub w2, w2, #4 // Need 65536 bytes of range. 447 ldr w10, [x1, x2] 448 str w10, [x9, x2] 449 cbnz w2, 1b 450 4512: 452 // Store null into ArtMethod* at bottom of frame. 453 str xzr, [sp] 454.endm 455 456.macro INVOKE_STUB_CALL_AND_RETURN 457 458 REFRESH_MARKING_REGISTER 459 REFRESH_SUSPEND_CHECK_REGISTER 460 461 // load method-> METHOD_QUICK_CODE_OFFSET 462 ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 463 // Branch to method. 464 blr x9 465 466 // Pop the ArtMethod* (null), arguments and alignment padding from the stack. 467 mov sp, xFP 468 .cfi_def_cfa_register sp 469 470 // Restore saved registers including value address and shorty address. 471 RESTORE_REG x19, 24 472 RESTORE_TWO_REGS x20, x21, 32 473 RESTORE_TWO_REGS xFP, xLR, 48 474 RESTORE_TWO_REGS_DECREASE_FRAME x4, x5, SAVE_SIZE 475 476 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 477 ldrb w10, [x5] 478 479 // Check the return type and store the correct register into the jvalue in memory. 480 // Use numeric label as this is a macro and Clang's assembler does not have unique-id variables. 481 482 // Don't set anything for a void type. 483 cmp w10, #'V' 484 beq 1f 485 486 // Is it a double? 487 cmp w10, #'D' 488 beq 2f 489 490 // Is it a float? 491 cmp w10, #'F' 492 beq 3f 493 494 // Just store x0. Doesn't matter if it is 64 or 32 bits. 495 str x0, [x4] 496 4971: // Finish up. 498 ret 499 5002: // Store double. 501 str d0, [x4] 502 ret 503 5043: // Store float. 505 str s0, [x4] 506 ret 507 508.endm 509 510 511// Macro for loading an argument into a register. 512// label - the base name of the label of the load routine, 513// reg - the register to load, 514// args - pointer to current argument, incremented by size, 515// size - the size of the register - 4 or 8 bytes, 516// nh4_reg - the register to fill with the address of the next handler for 4-byte values, 517// nh4_l - the base name of the label of the next handler for 4-byte values, 518// nh8_reg - the register to fill with the address of the next handler for 8-byte values, 519// nh8_l - the base name of the label of the next handler for 8-byte values, 520// cont - the base name of the label for continuing the shorty processing loop, 521// suffix - suffix added to all labels to make labels unique for different users. 522.macro INVOKE_STUB_LOAD_REG label, reg, args, size, nh4_reg, nh4_l, nh8_reg, nh8_l, cont, suffix 523\label\suffix: 524 ldr \reg, [\args], #\size 525 adr \nh4_reg, \nh4_l\suffix 526 adr \nh8_reg, \nh8_l\suffix 527 b \cont\suffix 528.endm 529 530// Macro for skipping an argument that does not fit into argument registers. 531// label - the base name of the label of the skip routine, 532// args - pointer to current argument, incremented by size, 533// size - the size of the argument - 4 or 8 bytes, 534// cont - the base name of the label for continuing the shorty processing loop, 535// suffix - suffix added to all labels to make labels unique for different users. 536.macro INVOKE_STUB_SKIP_ARG label, args, size, cont, suffix 537\label\suffix: 538 add \args, \args, #\size 539 b \cont\suffix 540.endm 541 542// Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 543// Parse the passed shorty to determine which register to load. 544// x5 - shorty, 545// x9 - points to arguments on the stack, 546// suffix - suffix added to all labels to make labels unique for different users. 547.macro INVOKE_STUB_LOAD_ALL_ARGS suffix 548 add x10, x5, #1 // Load shorty address, plus one to skip the return type. 549 550 // Load this (if instance method) and addresses for routines that load WXSD registers. 551 .ifc \suffix, _instance 552 ldr w1, [x9], #4 // Load "this" parameter, and increment arg pointer. 553 adr x11, .Lload_w2\suffix 554 adr x12, .Lload_x2\suffix 555 .else 556 adr x11, .Lload_w1\suffix 557 adr x12, .Lload_x1\suffix 558 .endif 559 adr x13, .Lload_s0\suffix 560 adr x14, .Lload_d0\suffix 561 562 // Loop to fill registers. 563.Lfill_regs\suffix: 564 ldrb w17, [x10], #1 // Load next character in signature, and increment. 565 cbz w17, .Lcall_method\suffix // Exit at end of signature. Shorty 0 terminated. 566 567 cmp w17, #'J' // Is this a long? 568 beq .Lload_long\suffix 569 570 cmp w17, #'F' // Is this a float? 571 beq .Lload_float\suffix 572 573 cmp w17, #'D' // Is this a double? 574 beq .Lload_double\suffix 575 576 // Everything else uses a 4-byte GPR. 577 br x11 578 579.Lload_long\suffix: 580 br x12 581 582.Lload_float\suffix: 583 br x13 584 585.Lload_double\suffix: 586 br x14 587 588// Handlers for loading other args (not float/double/long) into W registers. 589 .ifnc \suffix, _instance 590 INVOKE_STUB_LOAD_REG \ 591 .Lload_w1, w1, x9, 4, x11, .Lload_w2, x12, .Lload_x2, .Lfill_regs, \suffix 592 .endif 593 INVOKE_STUB_LOAD_REG .Lload_w2, w2, x9, 4, x11, .Lload_w3, x12, .Lload_x3, .Lfill_regs, \suffix 594 INVOKE_STUB_LOAD_REG .Lload_w3, w3, x9, 4, x11, .Lload_w4, x12, .Lload_x4, .Lfill_regs, \suffix 595 INVOKE_STUB_LOAD_REG .Lload_w4, w4, x9, 4, x11, .Lload_w5, x12, .Lload_x5, .Lfill_regs, \suffix 596 INVOKE_STUB_LOAD_REG .Lload_w5, w5, x9, 4, x11, .Lload_w6, x12, .Lload_x6, .Lfill_regs, \suffix 597 INVOKE_STUB_LOAD_REG .Lload_w6, w6, x9, 4, x11, .Lload_w7, x12, .Lload_x7, .Lfill_regs, \suffix 598 INVOKE_STUB_LOAD_REG .Lload_w7, w7, x9, 4, x11, .Lskip4, x12, .Lskip8, .Lfill_regs, \suffix 599 600// Handlers for loading longs into X registers. 601 .ifnc \suffix, _instance 602 INVOKE_STUB_LOAD_REG \ 603 .Lload_x1, x1, x9, 8, x11, .Lload_w2, x12, .Lload_x2, .Lfill_regs, \suffix 604 .endif 605 INVOKE_STUB_LOAD_REG .Lload_x2, x2, x9, 8, x11, .Lload_w3, x12, .Lload_x3, .Lfill_regs, \suffix 606 INVOKE_STUB_LOAD_REG .Lload_x3, x3, x9, 8, x11, .Lload_w4, x12, .Lload_x4, .Lfill_regs, \suffix 607 INVOKE_STUB_LOAD_REG .Lload_x4, x4, x9, 8, x11, .Lload_w5, x12, .Lload_x5, .Lfill_regs, \suffix 608 INVOKE_STUB_LOAD_REG .Lload_x5, x5, x9, 8, x11, .Lload_w6, x12, .Lload_x6, .Lfill_regs, \suffix 609 INVOKE_STUB_LOAD_REG .Lload_x6, x6, x9, 8, x11, .Lload_w7, x12, .Lload_x7, .Lfill_regs, \suffix 610 INVOKE_STUB_LOAD_REG .Lload_x7, x7, x9, 8, x11, .Lskip4, x12, .Lskip8, .Lfill_regs, \suffix 611 612// Handlers for loading singles into S registers. 613 INVOKE_STUB_LOAD_REG .Lload_s0, s0, x9, 4, x13, .Lload_s1, x14, .Lload_d1, .Lfill_regs, \suffix 614 INVOKE_STUB_LOAD_REG .Lload_s1, s1, x9, 4, x13, .Lload_s2, x14, .Lload_d2, .Lfill_regs, \suffix 615 INVOKE_STUB_LOAD_REG .Lload_s2, s2, x9, 4, x13, .Lload_s3, x14, .Lload_d3, .Lfill_regs, \suffix 616 INVOKE_STUB_LOAD_REG .Lload_s3, s3, x9, 4, x13, .Lload_s4, x14, .Lload_d4, .Lfill_regs, \suffix 617 INVOKE_STUB_LOAD_REG .Lload_s4, s4, x9, 4, x13, .Lload_s5, x14, .Lload_d5, .Lfill_regs, \suffix 618 INVOKE_STUB_LOAD_REG .Lload_s5, s5, x9, 4, x13, .Lload_s6, x14, .Lload_d6, .Lfill_regs, \suffix 619 INVOKE_STUB_LOAD_REG .Lload_s6, s6, x9, 4, x13, .Lload_s7, x14, .Lload_d7, .Lfill_regs, \suffix 620 INVOKE_STUB_LOAD_REG .Lload_s7, s7, x9, 4, x13, .Lskip4, x14, .Lskip8, .Lfill_regs, \suffix 621 622// Handlers for loading doubles into D registers. 623 INVOKE_STUB_LOAD_REG .Lload_d0, d0, x9, 8, x13, .Lload_s1, x14, .Lload_d1, .Lfill_regs, \suffix 624 INVOKE_STUB_LOAD_REG .Lload_d1, d1, x9, 8, x13, .Lload_s2, x14, .Lload_d2, .Lfill_regs, \suffix 625 INVOKE_STUB_LOAD_REG .Lload_d2, d2, x9, 8, x13, .Lload_s3, x14, .Lload_d3, .Lfill_regs, \suffix 626 INVOKE_STUB_LOAD_REG .Lload_d3, d3, x9, 8, x13, .Lload_s4, x14, .Lload_d4, .Lfill_regs, \suffix 627 INVOKE_STUB_LOAD_REG .Lload_d4, d4, x9, 8, x13, .Lload_s5, x14, .Lload_d5, .Lfill_regs, \suffix 628 INVOKE_STUB_LOAD_REG .Lload_d5, d5, x9, 8, x13, .Lload_s6, x14, .Lload_d6, .Lfill_regs, \suffix 629 INVOKE_STUB_LOAD_REG .Lload_d6, d6, x9, 8, x13, .Lload_s7, x14, .Lload_d7, .Lfill_regs, \suffix 630 INVOKE_STUB_LOAD_REG .Lload_d7, d7, x9, 8, x13, .Lskip4, x14, .Lskip8, .Lfill_regs, \suffix 631 632// Handlers for skipping arguments that do not fit into registers. 633 INVOKE_STUB_SKIP_ARG .Lskip4, x9, 4, .Lfill_regs, \suffix 634 INVOKE_STUB_SKIP_ARG .Lskip8, x9, 8, .Lfill_regs, \suffix 635 636.Lcall_method\suffix: 637.endm 638 639/* 640 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 641 * uint32_t *args, x1 642 * uint32_t argsize, w2 643 * Thread *self, x3 644 * JValue *result, x4 645 * char *shorty); x5 646 * +----------------------+ 647 * | | 648 * | C/C++ frame | 649 * | LR'' | 650 * | FP'' | <- SP' 651 * +----------------------+ 652 * +----------------------+ 653 * | x28 | <- TODO: Remove callee-saves. 654 * | : | 655 * | x19 | 656 * | SP' | 657 * | X5 | 658 * | X4 | Saved registers 659 * | LR' | 660 * | FP' | <- FP 661 * +----------------------+ 662 * | uint32_t out[n-1] | 663 * | : : | Outs 664 * | uint32_t out[0] | 665 * | ArtMethod* | <- SP value=null 666 * +----------------------+ 667 * 668 * Outgoing registers: 669 * x0 - Method* 670 * x1-x7 - integer parameters. 671 * d0-d7 - Floating point parameters. 672 * xSELF = self 673 * SP = & of ArtMethod* 674 * x1 = "this" pointer. 675 * 676 */ 677ENTRY art_quick_invoke_stub 678 // Spill registers as per AACPS64 calling convention. 679 INVOKE_STUB_CREATE_FRAME 680 681 // Load args into registers. 682 INVOKE_STUB_LOAD_ALL_ARGS _instance 683 684 // Call the method and return. 685 INVOKE_STUB_CALL_AND_RETURN 686END art_quick_invoke_stub 687 688/* extern"C" 689 * void art_quick_invoke_static_stub(ArtMethod *method, x0 690 * uint32_t *args, x1 691 * uint32_t argsize, w2 692 * Thread *self, x3 693 * JValue *result, x4 694 * char *shorty); x5 695 */ 696ENTRY art_quick_invoke_static_stub 697 // Spill registers as per AACPS64 calling convention. 698 INVOKE_STUB_CREATE_FRAME 699 700 // Load args into registers. 701 INVOKE_STUB_LOAD_ALL_ARGS _static 702 703 // Call the method and return. 704 INVOKE_STUB_CALL_AND_RETURN 705END art_quick_invoke_static_stub 706 707 708 709/* extern"C" void art_quick_osr_stub(void** stack, x0 710 * size_t stack_size_in_bytes, x1 711 * const uint8_t* native_pc, x2 712 * JValue *result, x3 713 * char *shorty, x4 714 * Thread *self) x5 715 */ 716ENTRY art_quick_osr_stub 717 SAVE_SIZE=22*8 718 SAVE_TWO_REGS_INCREASE_FRAME x3, x4, SAVE_SIZE 719 SAVE_TWO_REGS x19, x20, 16 720 SAVE_TWO_REGS x21, x22, 32 721 SAVE_TWO_REGS x23, x24, 48 722 SAVE_TWO_REGS x25, x26, 64 723 SAVE_TWO_REGS x27, x28, 80 724 SAVE_TWO_REGS xFP, xLR, 96 725 stp d8, d9, [sp, #112] 726 stp d10, d11, [sp, #128] 727 stp d12, d13, [sp, #144] 728 stp d14, d15, [sp, #160] 729 730 mov xSELF, x5 // Move thread pointer into SELF register. 731 REFRESH_MARKING_REGISTER 732 REFRESH_SUSPEND_CHECK_REGISTER 733 734 INCREASE_FRAME 16 735 str xzr, [sp] // Store null for ArtMethod* slot 736 // Branch to stub. 737 CFI_REMEMBER_STATE 738 bl .Losr_entry 739 DECREASE_FRAME 16 740 741 // Restore saved registers including value address and shorty address. 742 ldp d8, d9, [sp, #112] 743 ldp d10, d11, [sp, #128] 744 ldp d12, d13, [sp, #144] 745 ldp d14, d15, [sp, #160] 746 RESTORE_TWO_REGS x19, x20, 16 747 RESTORE_TWO_REGS x21, x22, 32 748 RESTORE_TWO_REGS x23, x24, 48 749 RESTORE_TWO_REGS x25, x26, 64 750 RESTORE_TWO_REGS x27, x28, 80 751 RESTORE_TWO_REGS xFP, xLR, 96 752 RESTORE_TWO_REGS_DECREASE_FRAME x3, x4, SAVE_SIZE 753 754 // The compiler put the result in x0. Doesn't matter if it is 64 or 32 bits. 755 str x0, [x3] 756 ret 757 758.Losr_entry: 759 CFI_RESTORE_STATE_AND_DEF_CFA sp, (SAVE_SIZE+16) 760 761 mov x9, sp // Save stack pointer. 762 .cfi_def_cfa_register x9 763 764 // Update stack pointer for the callee 765 sub sp, sp, x1 766 767 // Update link register slot expected by the callee. 768 sub w1, w1, #8 769 str lr, [sp, x1] 770 771 // Copy arguments into stack frame. 772 // Use simple copy routine for now. 773 // 4 bytes per slot. 774 // X0 - source address 775 // W1 - args length 776 // SP - destination address. 777 // W10 - temporary 778.Losr_loop_entry: 779 cbz w1, .Losr_loop_exit 780 sub w1, w1, #4 781 ldr w10, [x0, x1] 782 str w10, [sp, x1] 783 b .Losr_loop_entry 784 785.Losr_loop_exit: 786 // Branch to the OSR entry point. 787 br x2 788 789END art_quick_osr_stub 790 791 /* 792 * On entry x0 is the long jump context. This is expected to be returned from a previous 793 * entrypoint call which threw an exception or deoptimized. 794 * IP0 and IP1 shall be clobbered rather than retrieved from gprs_. 795 */ 796 797ENTRY art_quick_do_long_jump 798 // Reserve space for the gprs + fprs; 799 INCREASE_FRAME ARM64_LONG_JUMP_CONTEXT_SIZE 800 801 mov x1, sp 802 add x2, sp, #ARM64_LONG_JUMP_GPRS_SIZE 803 804 bl artContextCopyForLongJump // Context* context, uintptr_t* gprs, uintptr_t* fprs 805 806 add x0, sp, #ARM64_LONG_JUMP_GPRS_SIZE 807 808 // Load FPRs 809 ldp d0, d1, [x0, #0] 810 ldp d2, d3, [x0, #16] 811 ldp d4, d5, [x0, #32] 812 ldp d6, d7, [x0, #48] 813 ldp d8, d9, [x0, #64] 814 ldp d10, d11, [x0, #80] 815 ldp d12, d13, [x0, #96] 816 ldp d14, d15, [x0, #112] 817 ldp d16, d17, [x0, #128] 818 ldp d18, d19, [x0, #144] 819 ldp d20, d21, [x0, #160] 820 ldp d22, d23, [x0, #176] 821 ldp d24, d25, [x0, #192] 822 ldp d26, d27, [x0, #208] 823 ldp d28, d29, [x0, #224] 824 ldp d30, d31, [x0, #240] 825 826 ldp x0, x1, [sp, #0] 827 ldp x2, x3, [sp, #16] 828 ldp x4, x5, [sp, #32] 829 ldp x6, x7, [sp, #48] 830 ldp x8, x9, [sp, #64] 831 ldp x10, x11, [sp, #80] 832 ldp x12, x13, [sp, #96] 833 ldp x14, x15, [sp, #112] 834 // Do not load IP0 (x16) and IP1 (x17), these shall be clobbered below. 835 // Don't load the platform register (x18) either. 836 ldr x19, [sp, #152] // xSELF. 837 ldp x20, x21, [sp, #160] // For Baker RB, wMR (w20) is reloaded below. 838 ldp x22, x23, [sp, #176] 839 ldp x24, x25, [sp, #192] 840 ldp x26, x27, [sp, #208] 841 ldp x28, x29, [sp, #224] 842 ldp x30, xIP0, [sp, #240] // LR and SP, load SP to IP0. 843 844 // Load PC to IP1, it's at the end (after the space for the unused XZR). 845 ldr xIP1, [sp, #33*8] 846 847 // Set SP. Do not access fprs_ and gprs_ from now, they are below SP. 848 mov sp, xIP0 849 .cfi_def_cfa_offset 0 850 851 REFRESH_MARKING_REGISTER 852 REFRESH_SUSPEND_CHECK_REGISTER 853 854 br xIP1 855END art_quick_do_long_jump 856 857 /* 858 * Entry from managed code that tries to lock the object in a fast path and 859 * calls `artLockObjectFromCode()` for the difficult cases, may block for GC. 860 * x0 holds the possibly null object to lock. 861 */ 862ENTRY art_quick_lock_object 863 LOCK_OBJECT_FAST_PATH x0, art_quick_lock_object_no_inline, /*can_be_null*/ 1 864END art_quick_lock_object 865 866 /* 867 * Entry from managed code that calls `artLockObjectFromCode()`, may block for GC. 868 * x0 holds the possibly null object to lock. 869 */ 870 .extern artLockObjectFromCode 871ENTRY art_quick_lock_object_no_inline 872 // This is also the slow path for art_quick_lock_object. 873 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 874 mov x1, xSELF // pass Thread::Current 875 bl artLockObjectFromCode // (Object* obj, Thread*) 876 RESTORE_SAVE_REFS_ONLY_FRAME 877 REFRESH_MARKING_REGISTER 878 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 879END art_quick_lock_object_no_inline 880 881 /* 882 * Entry from managed code that tries to unlock the object in a fast path and calls 883 * `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure. 884 * x0 holds the possibly null object to unlock. 885 */ 886ENTRY art_quick_unlock_object 887 UNLOCK_OBJECT_FAST_PATH x0, art_quick_unlock_object_no_inline, /*can_be_null*/ 1 888END art_quick_unlock_object 889 890 /* 891 * Entry from managed code that calls `artUnlockObjectFromCode()` 892 * and delivers exception on failure. 893 * x0 holds the possibly null object to unlock. 894 */ 895 .extern artUnlockObjectFromCode 896ENTRY art_quick_unlock_object_no_inline 897 // This is also the slow path for art_quick_unlock_object. 898 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC 899 mov x1, xSELF // pass Thread::Current 900 bl artUnlockObjectFromCode // (Object* obj, Thread*) 901 RESTORE_SAVE_REFS_ONLY_FRAME 902 REFRESH_MARKING_REGISTER 903 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 904END art_quick_unlock_object_no_inline 905 906 /* 907 * Entry from managed code that calls artInstanceOfFromCode and on failure calls 908 * artThrowClassCastExceptionForObject. 909 */ 910 .extern artInstanceOfFromCode 911 .extern artThrowClassCastExceptionForObject 912ENTRY art_quick_check_instance_of 913 // Type check using the bit string passes null as the target class. In that case just throw. 914 cbz x1, .Lthrow_class_cast_exception_for_bitstring_check 915 916 // Store arguments and link register 917 // Stack needs to be 16B aligned on calls. 918 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 919 SAVE_REG xLR, 24 920 921 // Call runtime code 922 bl artInstanceOfFromCode 923 924 // Restore LR. 925 RESTORE_REG xLR, 24 926 927 // Check for exception 928 CFI_REMEMBER_STATE 929 cbz x0, .Lthrow_class_cast_exception 930 931 // Restore and return 932 // TODO: We do not need to restore X0 and X1 on success. We also do not need 933 // to record CFI for them as the information is not very useful. 934 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 935 ret 936 937.Lthrow_class_cast_exception: 938 CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 939 // Restore 940 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 941 942.Lthrow_class_cast_exception_for_bitstring_check: 943 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 944 mov x2, xSELF // pass Thread::Current 945 bl artThrowClassCastExceptionForObject // (Object*, Class*, Thread*) 946 bl art_quick_do_long_jump // (Context*) 947 brk 0 // Unreached 948END art_quick_check_instance_of 949 950// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. 951.macro POP_REG_NE xReg, offset, xExclude 952 .ifnc \xReg, \xExclude 953 ldr \xReg, [sp, #\offset] // restore xReg 954 .cfi_restore \xReg 955 .endif 956.endm 957 958// Restore xReg1's value from [sp, #offset] if xReg1 is not the same as xExclude. 959// Restore xReg2's value from [sp, #(offset + 8)] if xReg2 is not the same as xExclude. 960.macro POP_REGS_NE xReg1, xReg2, offset, xExclude 961 .ifc \xReg1, \xExclude 962 ldr \xReg2, [sp, #(\offset + 8)] // restore xReg2 963 .else 964 .ifc \xReg2, \xExclude 965 ldr \xReg1, [sp, #\offset] // restore xReg1 966 .else 967 ldp \xReg1, \xReg2, [sp, #\offset] // restore xReg1 and xReg2 968 .endif 969 .endif 970 .cfi_restore \xReg1 971 .cfi_restore \xReg2 972.endm 973 974 // Helper macros for `art_quick_aput_obj`. 975#ifdef USE_READ_BARRIER 976#ifdef USE_BAKER_READ_BARRIER 977.macro BAKER_RB_CHECK_GRAY_BIT_AND_LOAD wDest, xObj, offset, gray_slow_path_label 978 ldr wIP0, [\xObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 979 tbnz wIP0, #LOCK_WORD_READ_BARRIER_STATE_SHIFT, \gray_slow_path_label 980 // False dependency to avoid needing load/load fence. 981 add \xObj, \xObj, xIP0, lsr #32 982 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b; zero-extends to xN. 983 UNPOISON_HEAP_REF \wDest 984.endm 985 986.macro BAKER_RB_LOAD_AND_MARK wDest, xObj, offset, mark_function 987 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b; zero-extends to xN. 988 UNPOISON_HEAP_REF \wDest 989 // Save LR in a register preserved by `art_quick_read_barrier_mark_regNN` 990 // and unused by the `art_quick_aput_obj`. 991 mov x5, lr 992 bl \mark_function 993 mov lr, x5 // Restore LR. 994.endm 995#else // USE_BAKER_READ_BARRIER 996 .extern artReadBarrierSlow 997.macro READ_BARRIER_SLOW xDest, wDest, xObj, offset 998 // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. 999 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 48 1000 SAVE_TWO_REGS x2, x3, 16 1001 SAVE_TWO_REGS x4, xLR, 32 1002 1003 // mov x0, \xRef // pass ref in x0 (no-op for now since parameter ref is unused) 1004 .ifnc \xObj, x1 1005 mov x1, \xObj // pass xObj 1006 .endif 1007 mov w2, #\offset // pass offset 1008 bl artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) 1009 // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. 1010 .ifnc \wDest, w0 1011 mov \wDest, w0 // save return value in wDest 1012 .endif 1013 1014 // Conditionally restore saved registers 1015 POP_REG_NE x0, 0, \xDest 1016 POP_REG_NE x1, 8, \xDest 1017 POP_REG_NE x2, 16, \xDest 1018 POP_REG_NE x3, 24, \xDest 1019 POP_REG_NE x4, 32, \xDest 1020 RESTORE_REG xLR, 40 1021 DECREASE_FRAME 48 1022.endm 1023#endif // USE_BAKER_READ_BARRIER 1024#endif // USE_READ_BARRIER 1025 1026ENTRY art_quick_aput_obj 1027 cbz x2, .Laput_obj_null 1028#if defined(USE_READ_BARRIER) && !defined(USE_BAKER_READ_BARRIER) 1029 READ_BARRIER_SLOW x3, w3, x0, MIRROR_OBJECT_CLASS_OFFSET 1030 READ_BARRIER_SLOW x3, w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET 1031 READ_BARRIER_SLOW x4, w4, x2, MIRROR_OBJECT_CLASS_OFFSET 1032#else // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 1033#ifdef USE_READ_BARRIER 1034 cbnz wMR, .Laput_obj_gc_marking 1035#endif // USE_READ_BARRIER 1036 ldr w3, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b; zero-extends to x3. 1037 UNPOISON_HEAP_REF w3 1038 ldr w3, [x3, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Heap reference = 32b; zero-extends to x3. 1039 UNPOISON_HEAP_REF w3 1040 ldr w4, [x2, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b; zero-extends to x4. 1041 UNPOISON_HEAP_REF w4 1042#endif // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 1043 cmp w3, w4 // value's type == array's component type - trivial assignability 1044 bne .Laput_obj_check_assignability 1045.Laput_obj_store: 1046 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1047 POISON_HEAP_REF w2 1048 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1049 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1050 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1051 strb w3, [x3, x0] 1052 ret 1053 1054.Laput_obj_null: 1055 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1056 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1057 ret 1058 1059.Laput_obj_check_assignability: 1060 // Store arguments and link register 1061 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1062 SAVE_TWO_REGS x2, xLR, 16 1063 1064 // Call runtime code 1065 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1066 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1067 bl artIsAssignableFromCode 1068 1069 // Check for exception 1070 CFI_REMEMBER_STATE 1071 cbz x0, .Laput_obj_throw_array_store_exception 1072 1073 // Restore 1074 RESTORE_TWO_REGS x2, xLR, 16 1075 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1076 1077 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1078 POISON_HEAP_REF w2 1079 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1080 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1081 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1082 strb w3, [x3, x0] 1083 ret 1084 1085.Laput_obj_throw_array_store_exception: 1086 CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 1087 RESTORE_TWO_REGS x2, xLR, 16 1088 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1089 1090#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1091 CFI_REMEMBER_STATE 1092#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1093 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 1094 mov x1, x2 // Pass value. 1095 mov x2, xSELF // Pass Thread::Current. 1096 bl artThrowArrayStoreException // (Object*, Object*, Thread*). 1097 bl art_quick_do_long_jump // (Context*) 1098 brk 0 // Unreached 1099 1100#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1101 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 1102.Laput_obj_gc_marking: 1103 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1104 w3, x0, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_array_class 1105.Laput_obj_mark_array_class_continue: 1106 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1107 w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, .Laput_obj_mark_array_element 1108.Laput_obj_mark_array_element_continue: 1109 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1110 w4, x2, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_object_class 1111.Laput_obj_mark_object_class_continue: 1112 cmp w3, w4 // value's type == array's component type - trivial assignability 1113 bne .Laput_obj_check_assignability 1114 b .Laput_obj_store 1115 1116.Laput_obj_mark_array_class: 1117 BAKER_RB_LOAD_AND_MARK w3, x0, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg03 1118 b .Laput_obj_mark_array_class_continue 1119 1120.Laput_obj_mark_array_element: 1121 BAKER_RB_LOAD_AND_MARK \ 1122 w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, art_quick_read_barrier_mark_reg03 1123 b .Laput_obj_mark_array_element_continue 1124 1125.Laput_obj_mark_object_class: 1126 BAKER_RB_LOAD_AND_MARK w4, x2, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg04 1127 b .Laput_obj_mark_object_class_continue 1128#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1129END art_quick_aput_obj 1130 1131// Macro to facilitate adding new allocation entrypoints. 1132.macro ONE_ARG_DOWNCALL name, entrypoint, return 1133 .extern \entrypoint 1134ENTRY \name 1135 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1136 mov x1, xSELF // pass Thread::Current 1137 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1138 RESTORE_SAVE_REFS_ONLY_FRAME 1139 REFRESH_MARKING_REGISTER 1140 \return 1141END \name 1142.endm 1143 1144// Macro to facilitate adding new allocation entrypoints. 1145.macro TWO_ARG_DOWNCALL name, entrypoint, return 1146 .extern \entrypoint 1147ENTRY \name 1148 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1149 mov x2, xSELF // pass Thread::Current 1150 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1151 RESTORE_SAVE_REFS_ONLY_FRAME 1152 REFRESH_MARKING_REGISTER 1153 \return 1154END \name 1155.endm 1156 1157// Macro to facilitate adding new allocation entrypoints. 1158.macro THREE_ARG_DOWNCALL name, entrypoint, return 1159 .extern \entrypoint 1160ENTRY \name 1161 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1162 mov x3, xSELF // pass Thread::Current 1163 bl \entrypoint 1164 RESTORE_SAVE_REFS_ONLY_FRAME 1165 REFRESH_MARKING_REGISTER 1166 \return 1167END \name 1168.endm 1169 1170// Macro to facilitate adding new allocation entrypoints. 1171.macro FOUR_ARG_DOWNCALL name, entrypoint, return 1172 .extern \entrypoint 1173ENTRY \name 1174 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1175 mov x4, xSELF // pass Thread::Current 1176 bl \entrypoint // 1177 RESTORE_SAVE_REFS_ONLY_FRAME 1178 REFRESH_MARKING_REGISTER 1179 \return 1180END \name 1181.endm 1182 1183 /* 1184 * Macro for resolution and initialization of indexed DEX file 1185 * constants such as classes and strings. 1186 */ 1187.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1188 name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 1189 .extern \entrypoint 1190ENTRY \name 1191 SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for stack crawl 1192 mov x1, xSELF // pass Thread::Current 1193 bl \entrypoint // (int32_t index, Thread* self) 1194 cbz w0, 1f // If result is null, deliver the OOME. 1195 DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 x1, /* is_ref= */ 1 11961: 1197 DELIVER_PENDING_EXCEPTION_FRAME_READY 1198END \name 1199.endm 1200 1201.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint 1202 ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1203 \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 1204.endm 1205 1206.macro RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1207 CFI_REMEMBER_STATE 1208 cbz w0, 1f // result zero branch over 1209 DEOPT_OR_RETURN x1, /*is_ref=*/1 // check for deopt or return 12101: 1211 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 1212 DELIVER_PENDING_EXCEPTION 1213.endm 1214 1215 1216 /* 1217 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 1218 * failure. 1219 */ 1220TWO_ARG_DOWNCALL art_quick_handle_fill_data, \ 1221 artHandleFillArrayDataFromCode, \ 1222 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 1223 1224 /* 1225 * Entry from managed code when uninitialized static storage, this stub will run the class 1226 * initializer and deliver the exception on error. On success the static storage base is 1227 * returned. 1228 */ 1229ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT \ 1230 art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 1231ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode 1232ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1233 art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode 1234ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode 1235ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode 1236ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 1237 1238GENERATE_FIELD_ENTRYPOINTS 1239 1240// Generate the allocation entrypoints for each allocator. 1241GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 1242// Comment out allocators that have arm64 specific asm. 1243// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1244// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1245GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1246GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB) 1247// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1248// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 1249// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 1250// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 1251// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 1252GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1253GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1254GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1255 1256// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 1257// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 1258GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 1259GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB) 1260// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 1261// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 1262// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 1263// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 1264// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 1265GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 1266GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 1267GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 1268 1269// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1270// If isInitialized=0 the compiler can only assume it's been at least resolved. 1271.macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name, isInitialized 1272ENTRY \c_name 1273 // Fast path rosalloc allocation. 1274 // x0: type, xSELF(x19): Thread::Current 1275 // x1-x7: free. 1276 ldr x3, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1277 // allocation stack has room. 1278 // ldp won't work due to large offset. 1279 ldr x4, [xSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1280 cmp x3, x4 1281 bhs .Lslow_path\c_name 1282 ldr w3, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x3) 1283 cmp x3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1284 // local allocation. 1285 // If the class is not yet visibly initialized, or it is finalizable, 1286 // the object size will be very large to force the branch below to be taken. 1287 // 1288 // See Class::SetStatus() in class.cc for more details. 1289 bhs .Lslow_path\c_name 1290 // Compute the rosalloc bracket index 1291 // from the size. Since the size is 1292 // already aligned we can combine the 1293 // two shifts together. 1294 add x4, xSELF, x3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1295 // Subtract pointer size since there 1296 // are no runs for 0 byte allocations 1297 // and the size is already aligned. 1298 ldr x4, [x4, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1299 // Load the free list head (x3). This 1300 // will be the return val. 1301 ldr x3, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1302 cbz x3, .Lslow_path\c_name 1303 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1304 ldr x1, [x3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1305 // and update the list head with the 1306 // next pointer. 1307 str x1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1308 // Store the class pointer in the 1309 // header. This also overwrites the 1310 // next pointer. The offsets are 1311 // asserted to match. 1312 1313#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1314#error "Class pointer needs to overwrite next pointer." 1315#endif 1316 POISON_HEAP_REF w0 1317 str w0, [x3, #MIRROR_OBJECT_CLASS_OFFSET] 1318 // Push the new object onto the thread 1319 // local allocation stack and 1320 // increment the thread local 1321 // allocation stack top. 1322 ldr x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1323 str w3, [x1], #COMPRESSED_REFERENCE_SIZE // (Increment x1 as a side effect.) 1324 str x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1325 // Decrement the size of the free list 1326 1327 // After this "STR" the object is published to the thread local allocation stack, 1328 // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. 1329 // It is not yet visible to the running (user) compiled code until after the return. 1330 // 1331 // To avoid the memory barrier prior to the "STR", a trick is employed, by differentiating 1332 // the state of the allocation stack slot. It can be a pointer to one of: 1333 // 0) Null entry, because the stack was bumped but the new pointer wasn't written yet. 1334 // (The stack initial state is "null" pointers). 1335 // 1) A partially valid object, with an invalid class pointer to the next free rosalloc slot. 1336 // 2) A fully valid object, with a valid class pointer pointing to a real class. 1337 // Other states are not allowed. 1338 // 1339 // An object that is invalid only temporarily, and will eventually become valid. 1340 // The internal runtime code simply checks if the object is not null or is partial and then 1341 // ignores it. 1342 // 1343 // (Note: The actual check is done by seeing if a non-null object has a class pointer pointing 1344 // to ClassClass, and that the ClassClass's class pointer is self-cyclic. A rosalloc free slot 1345 // "next" pointer is not-cyclic.) 1346 // 1347 // See also b/28790624 for a listing of CLs dealing with this race. 1348 ldr w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1349 sub x1, x1, #1 1350 // TODO: consider combining this store 1351 // and the list head store above using 1352 // strd. 1353 str w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1354 1355 mov x0, x3 // Set the return value and return. 1356 // No barrier. The class is already observably initialized (otherwise the fast 1357 // path size check above would fail) and new-instance allocations are protected 1358 // from publishing by the compiler which inserts its own StoreStore barrier. 1359 ret 1360.Lslow_path\c_name: 1361 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1362 mov x1, xSELF // pass Thread::Current 1363 bl \cxx_name 1364 RESTORE_SAVE_REFS_ONLY_FRAME 1365 REFRESH_MARKING_REGISTER 1366 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1367END \c_name 1368.endm 1369 1370ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, \ 1371 artAllocObjectFromCodeResolvedRosAlloc, /* isInitialized */ 0 1372ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, \ 1373 artAllocObjectFromCodeInitializedRosAlloc, /* isInitialized */ 1 1374 1375// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1376// If isInitialized=0 the compiler can only assume it's been at least resolved. 1377.macro ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED slowPathLabel isInitialized 1378 ldr x4, [xSELF, #THREAD_LOCAL_POS_OFFSET] 1379 ldr x5, [xSELF, #THREAD_LOCAL_END_OFFSET] 1380 ldr w7, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x7). 1381 add x6, x4, x7 // Add object size to tlab pos. 1382 cmp x6, x5 // Check if it fits, overflow works 1383 // since the tlab pos and end are 32 1384 // bit values. 1385 1386 // If the class is not yet visibly initialized, or it is finalizable, 1387 // the object size will be very large to force the branch below to be taken. 1388 // 1389 // See Class::SetStatus() in class.cc for more details. 1390 bhi \slowPathLabel 1391 str x6, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1392 POISON_HEAP_REF w0 1393 str w0, [x4, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1394 mov x0, x4 1395 // No barrier. The class is already observably initialized (otherwise the fast 1396 // path size check above would fail) and new-instance allocations are protected 1397 // from publishing by the compiler which inserts its own StoreStore barrier. 1398 ret 1399.endm 1400 1401// The common code for art_quick_alloc_object_*region_tlab 1402// Currently the implementation ignores isInitialized. TODO(b/172087402): clean this up. 1403// Caller must execute a constructor fence after this. 1404.macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint, isInitialized 1405ENTRY \name 1406 // Fast path region tlab allocation. 1407 // x0: type, xSELF(x19): Thread::Current 1408 // x1-x7: free. 1409 ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED .Lslow_path\name, \isInitialized 1410.Lslow_path\name: 1411 SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. 1412 mov x1, xSELF // Pass Thread::Current. 1413 bl \entrypoint // (mirror::Class*, Thread*) 1414 RESTORE_SAVE_REFS_ONLY_FRAME 1415 REFRESH_MARKING_REGISTER 1416 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1417END \name 1418.endm 1419 1420GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1421 art_quick_alloc_object_resolved_region_tlab, \ 1422 artAllocObjectFromCodeResolvedRegionTLAB, /* isInitialized */ 0 1423GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1424 art_quick_alloc_object_initialized_region_tlab, \ 1425 artAllocObjectFromCodeInitializedRegionTLAB, /* isInitialized */ 1 1426GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1427 art_quick_alloc_object_resolved_tlab, \ 1428 artAllocObjectFromCodeResolvedTLAB, /* isInitialized */ 0 1429GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1430 art_quick_alloc_object_initialized_tlab, \ 1431 artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 1432 1433.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE \ 1434 slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1435 and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignment mask 1436 // (addr + 7) & ~7. The mask must 1437 // be 64 bits to keep high bits in 1438 // case of overflow. 1439 // Negative sized arrays are handled here since xCount holds a zero extended 32 bit value. 1440 // Negative ints become large 64 bit unsigned ints which will always be larger than max signed 1441 // 32 bit int. Since the max shift for arrays is 3, it can not become a negative 64 bit int. 1442 cmp \xTemp1, #MIN_LARGE_OBJECT_THRESHOLD // Possibly a large object, go slow 1443 bhs \slowPathLabel // path. 1444 1445 ldr \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Check tlab for space, note that 1446 // we use (end - begin) to handle 1447 // negative size arrays. It is 1448 // assumed that a negative size will 1449 // always be greater unsigned than 1450 // region size. 1451 ldr \xTemp2, [xSELF, #THREAD_LOCAL_END_OFFSET] 1452 sub \xTemp2, \xTemp2, \xTemp0 1453 cmp \xTemp1, \xTemp2 1454 1455 // The array class is always initialized here. Unlike new-instance, 1456 // this does not act as a double test. 1457 bhi \slowPathLabel 1458 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1459 // Move old thread_local_pos to x0 1460 // for the return value. 1461 mov x0, \xTemp0 1462 add \xTemp0, \xTemp0, \xTemp1 1463 str \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1464 POISON_HEAP_REF \wClass 1465 str \wClass, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1466 str \wCount, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1467// new-array is special. The class is loaded and immediately goes to the Initialized state 1468// before it is published. Therefore the only fence needed is for the publication of the object. 1469// See ClassLinker::CreateArrayClass() for more details. 1470 1471// For publication of the new array, we don't need a 'dmb ishst' here. 1472// The compiler generates 'dmb ishst' for all new-array insts. 1473 ret 1474.endm 1475 1476// Caller must execute a constructor fence after this. 1477.macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup 1478ENTRY \name 1479 // Fast path array allocation for region tlab allocation. 1480 // x0: mirror::Class* type 1481 // x1: int32_t component_count 1482 // x2-x7: free. 1483 mov x3, x0 1484 \size_setup x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1485 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE \ 1486 .Lslow_path\name, x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1487.Lslow_path\name: 1488 // x0: mirror::Class* klass 1489 // x1: int32_t component_count 1490 // x2: Thread* self 1491 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1492 mov x2, xSELF // pass Thread::Current 1493 bl \entrypoint 1494 RESTORE_SAVE_REFS_ONLY_FRAME 1495 REFRESH_MARKING_REGISTER 1496 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1497END \name 1498.endm 1499 1500.macro COMPUTE_ARRAY_SIZE_UNKNOWN \ 1501 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1502 // Array classes are never finalizable or uninitialized, no need to check. 1503 ldr \wTemp0, [\xClass, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1504 UNPOISON_HEAP_REF \wTemp0 1505 ldr \wTemp0, [\xTemp0, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1506 lsr \xTemp0, \xTemp0, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1507 // bits. 1508 // xCount is holding a 32 bit value, 1509 // it can not overflow. 1510 lsl \xTemp1, \xCount, \xTemp0 // Calculate data size 1511 // Add array data offset and alignment. 1512 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1513#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1514#error Long array data offset must be 4 greater than int array data offset. 1515#endif 1516 1517 add \xTemp0, \xTemp0, #1 // Add 4 to the length only if the 1518 // component size shift is 3 1519 // (for 64 bit alignment). 1520 and \xTemp0, \xTemp0, #4 1521 add \xTemp1, \xTemp1, \xTemp0 1522.endm 1523 1524.macro COMPUTE_ARRAY_SIZE_8 \ 1525 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1526 // Add array data offset and alignment. 1527 add \xTemp1, \xCount, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1528.endm 1529 1530.macro COMPUTE_ARRAY_SIZE_16 \ 1531 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1532 lsl \xTemp1, \xCount, #1 1533 // Add array data offset and alignment. 1534 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1535.endm 1536 1537.macro COMPUTE_ARRAY_SIZE_32 \ 1538 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1539 lsl \xTemp1, \xCount, #2 1540 // Add array data offset and alignment. 1541 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1542.endm 1543 1544.macro COMPUTE_ARRAY_SIZE_64 \ 1545 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1546 lsl \xTemp1, \xCount, #3 1547 // Add array data offset and alignment. 1548 add \xTemp1, \xTemp1, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1549.endm 1550 1551// TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove 1552// the entrypoint once all backends have been updated to use the size variants. 1553GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, \ 1554 artAllocArrayFromCodeResolvedRegionTLAB, \ 1555 COMPUTE_ARRAY_SIZE_UNKNOWN 1556GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, \ 1557 artAllocArrayFromCodeResolvedRegionTLAB, \ 1558 COMPUTE_ARRAY_SIZE_8 1559GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, \ 1560 artAllocArrayFromCodeResolvedRegionTLAB, \ 1561 COMPUTE_ARRAY_SIZE_16 1562GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, \ 1563 artAllocArrayFromCodeResolvedRegionTLAB, \ 1564 COMPUTE_ARRAY_SIZE_32 1565GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, \ 1566 artAllocArrayFromCodeResolvedRegionTLAB, \ 1567 COMPUTE_ARRAY_SIZE_64 1568GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, \ 1569 artAllocArrayFromCodeResolvedTLAB, \ 1570 COMPUTE_ARRAY_SIZE_UNKNOWN 1571GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, \ 1572 artAllocArrayFromCodeResolvedTLAB, \ 1573 COMPUTE_ARRAY_SIZE_8 1574GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, \ 1575 artAllocArrayFromCodeResolvedTLAB, \ 1576 COMPUTE_ARRAY_SIZE_16 1577GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, \ 1578 artAllocArrayFromCodeResolvedTLAB, \ 1579 COMPUTE_ARRAY_SIZE_32 1580GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, \ 1581 artAllocArrayFromCodeResolvedTLAB, \ 1582 COMPUTE_ARRAY_SIZE_64 1583 1584 /* 1585 * Called by managed code when the thread has been asked to suspend. 1586 */ 1587 .extern artTestSuspendFromCode 1588ENTRY art_quick_test_suspend 1589 // Save callee saves for stack crawl. 1590 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 1591 mov x0, xSELF 1592 bl artTestSuspendFromCode // (Thread*) 1593 1594 CFI_REMEMBER_STATE 1595 cbnz x0, .Ltest_suspend_deoptimize 1596 1597 // Normal return. 1598 RESTORE_SAVE_EVERYTHING_FRAME 1599 REFRESH_MARKING_REGISTER 1600 REFRESH_SUSPEND_CHECK_REGISTER 1601 ret 1602 1603.Ltest_suspend_deoptimize: 1604 // Deoptimize. 1605 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 1606 bl art_quick_do_long_jump // (Context*) 1607 brk 0 // Unreached 1608END art_quick_test_suspend 1609 1610 /* 1611 * Redirection point from implicit suspend check fault handler. 1612 */ 1613 .extern artImplicitSuspendFromCode 1614ENTRY art_quick_implicit_suspend 1615 // Save callee saves for stack crawl. 1616 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 1617 mov x0, xSELF 1618 bl artImplicitSuspendFromCode // (Thread*) 1619 1620 CFI_REMEMBER_STATE 1621 cbnz x0, .Limplicit_suspend_deopt 1622 1623 RESTORE_SAVE_EVERYTHING_FRAME 1624 REFRESH_MARKING_REGISTER 1625 REFRESH_SUSPEND_CHECK_REGISTER 1626 br lr // Do not use RET as we do not enter the entrypoint with "BL". 1627 1628.Limplicit_suspend_deopt: 1629 // Deoptimize. 1630 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 1631 bl art_quick_do_long_jump // (Context*) 1632 brk 0 // Unreached 1633END art_quick_implicit_suspend 1634 1635 /* 1636 * Called by managed code that is attempting to call a method on a proxy class. On entry 1637 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 1638 * method agrees with a ref and args callee save frame. 1639 */ 1640 .extern artQuickProxyInvokeHandler 1641ENTRY art_quick_proxy_invoke_handler 1642 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 1643 mov x2, xSELF // pass Thread::Current 1644 mov x3, sp // pass SP 1645 bl artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 1646 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1647 CFI_REMEMBER_STATE 1648 cbnz x2, .Lexception_in_proxy // success if no exception is pending 1649 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame 1650 REFRESH_MARKING_REGISTER 1651 fmov d0, x0 // Store result in d0 in case it was float or double 1652 ret // return on success 1653.Lexception_in_proxy: 1654 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS 1655 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1656 DELIVER_PENDING_EXCEPTION 1657END art_quick_proxy_invoke_handler 1658 1659 /* 1660 * Called to resolve an imt conflict. 1661 * x0 is the conflict ArtMethod. 1662 * xIP1 is a hidden argument that holds the target interface method. 1663 * 1664 * Note that this stub writes to xIP0, xIP1, and x0. 1665 */ 1666ENTRY art_quick_imt_conflict_trampoline 1667 ldr xIP0, [x0, #ART_METHOD_JNI_OFFSET_64] // Load ImtConflictTable 1668 ldr x0, [xIP0] // Load first entry in ImtConflictTable. 1669.Limt_table_iterate: 1670 cmp x0, xIP1 1671 // Branch if found. Benchmarks have shown doing a branch here is better. 1672 beq .Limt_table_found 1673 // If the entry is null, the interface method is not in the ImtConflictTable. 1674 cbz x0, .Lconflict_trampoline 1675 // Iterate over the entries of the ImtConflictTable. 1676 ldr x0, [xIP0, #(2 * __SIZEOF_POINTER__)]! 1677 b .Limt_table_iterate 1678.Limt_table_found: 1679 // We successfully hit an entry in the table. Load the target method 1680 // and jump to it. 1681 ldr x0, [xIP0, #__SIZEOF_POINTER__] 1682 ldr xIP0, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1683 br xIP0 1684.Lconflict_trampoline: 1685 // Call the runtime stub to populate the ImtConflictTable and jump to the 1686 // resolved method. 1687 mov x0, xIP1 // Load interface method 1688 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1689END art_quick_imt_conflict_trampoline 1690 1691ENTRY art_quick_resolution_trampoline 1692 SETUP_SAVE_REFS_AND_ARGS_FRAME 1693 mov x2, xSELF 1694 mov x3, sp 1695 bl artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 1696 CFI_REMEMBER_STATE 1697 cbz x0, 1f 1698 mov xIP0, x0 // Remember returned code pointer in xIP0. 1699 ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 1700 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1701 REFRESH_MARKING_REGISTER 1702 br xIP0 17031: 1704 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS 1705 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1706 DELIVER_PENDING_EXCEPTION 1707END art_quick_resolution_trampoline 1708 1709/* 1710 * Generic JNI frame layout: 1711 * 1712 * #-------------------# 1713 * | | 1714 * | caller method... | 1715 * #-------------------# <--- SP on entry 1716 * | Return X30/LR | 1717 * | X29/FP | callee save 1718 * | X28 | callee save 1719 * | X27 | callee save 1720 * | X26 | callee save 1721 * | X25 | callee save 1722 * | X24 | callee save 1723 * | X23 | callee save 1724 * | X22 | callee save 1725 * | X21 | callee save 1726 * | X20 | callee save 1727 * | X7 | arg7 1728 * | X6 | arg6 1729 * | X5 | arg5 1730 * | X4 | arg4 1731 * | X3 | arg3 1732 * | X2 | arg2 1733 * | X1 | arg1 1734 * | D7 | float arg 8 1735 * | D6 | float arg 7 1736 * | D5 | float arg 6 1737 * | D4 | float arg 5 1738 * | D3 | float arg 4 1739 * | D2 | float arg 3 1740 * | D1 | float arg 2 1741 * | D0 | float arg 1 1742 * | padding | // 8B 1743 * | Method* | <- X0 (Managed frame similar to SaveRefsAndArgs.) 1744 * #-------------------# 1745 * | local ref cookie | // 4B 1746 * | padding | // 0B or 4B to align stack args on 8B address 1747 * #-------------------# 1748 * | JNI Stack Args | // Empty if all args fit into registers x0-x7, d0-d7. 1749 * #-------------------# <--- SP on native call (1) 1750 * | Free scratch | 1751 * #-------------------# 1752 * | SP for JNI call | // Pointer to (1). 1753 * #-------------------# 1754 * | Hidden arg | // For @CriticalNative 1755 * #-------------------# 1756 * | | 1757 * | Stack for Regs | The trampoline assembly will pop these values 1758 * | | into registers for native call 1759 * #-------------------# 1760 */ 1761 /* 1762 * Called to do a generic JNI down-call 1763 */ 1764 .extern artQuickGenericJniTrampoline 1765ENTRY art_quick_generic_jni_trampoline 1766 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 1767 1768 // Save SP, so we can have static CFI info. 1769 mov x28, sp 1770 .cfi_def_cfa_register x28 1771 1772 mov xIP0, #GENERIC_JNI_TRAMPOLINE_RESERVED_AREA 1773 sub sp, sp, xIP0 1774 1775 // prepare for artQuickGenericJniTrampoline call 1776 // (Thread*, managed_sp, reserved_area) 1777 // x0 x1 x2 <= C calling convention 1778 // xSELF x28 sp <= where they are 1779 1780 mov x0, xSELF // Thread* 1781 mov x1, x28 // SP for the managed frame. 1782 mov x2, sp // reserved area for arguments and other saved data (up to managed frame) 1783 bl artQuickGenericJniTrampoline // (Thread*, sp) 1784 1785 // The C call will have registered the complete save-frame on success. 1786 // The result of the call is: 1787 // x0: pointer to native code, 0 on error. 1788 // The bottom of the reserved area contains values for arg registers, 1789 // hidden arg register and SP for out args for the call. 1790 1791 // Check for error (class init check or locking for synchronized native method can throw). 1792 cbz x0, .Lexception_in_native 1793 1794 // Save the code pointer 1795 mov xIP0, x0 1796 1797 // Load parameters from frame into registers. 1798 ldp x0, x1, [sp] 1799 ldp x2, x3, [sp, #16] 1800 ldp x4, x5, [sp, #32] 1801 ldp x6, x7, [sp, #48] 1802 1803 ldp d0, d1, [sp, #64] 1804 ldp d2, d3, [sp, #80] 1805 ldp d4, d5, [sp, #96] 1806 ldp d6, d7, [sp, #112] 1807 1808 // Load hidden arg (x15) for @CriticalNative and SP for out args. 1809 ldp x15, xIP1, [sp, #128] 1810 1811 // Apply the new SP for out args, releasing unneeded reserved area. 1812 mov sp, xIP1 1813 1814 blr xIP0 // native call. 1815 1816 // result sign extension is handled in C code 1817 // prepare for artQuickGenericJniEndTrampoline call 1818 // (Thread*, result, result_f) 1819 // x0 x1 x2 <= C calling convention 1820 mov x1, x0 // Result (from saved). 1821 mov x0, xSELF // Thread register. 1822 fmov x2, d0 // d0 will contain floating point result, but needs to go into x2 1823 1824 bl artQuickGenericJniEndTrampoline 1825 1826 // Pending exceptions possible. 1827 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1828 cbnz x2, .Lexception_in_native 1829 1830 // Tear down the alloca. 1831 mov sp, x28 1832 1833 LOAD_RUNTIME_INSTANCE x1 1834 ldrb w1, [x1, #RUN_EXIT_HOOKS_OFFSET_FROM_RUNTIME_INSTANCE] 1835 CFI_REMEMBER_STATE 1836 cbnz w1, .Lcall_method_exit_hook 1837.Lcall_method_exit_hook_done: 1838 1839 // Tear down the callee-save frame. 1840 .cfi_def_cfa_register sp 1841 // Restore callee-saves and LR as in `RESTORE_SAVE_REFS_AND_ARGS_FRAME` 1842 // but do not restore argument registers. 1843 // Note: Likewise, we could avoid restoring X20 in the case of Baker 1844 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 1845 // later; but it's not worth handling this special case. 1846#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 1847#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 1848#endif 1849 RESTORE_REG x20, 136 1850 RESTORE_TWO_REGS x21, x22, 144 1851 RESTORE_TWO_REGS x23, x24, 160 1852 RESTORE_TWO_REGS x25, x26, 176 1853 RESTORE_TWO_REGS x27, x28, 192 1854 RESTORE_TWO_REGS x29, xLR, 208 1855 // Remove the frame. 1856 DECREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 1857 1858 REFRESH_MARKING_REGISTER 1859 1860 // store into fpr, for when it's a fpr return... 1861 fmov d0, x0 1862 ret 1863 1864.Lcall_method_exit_hook: 1865 CFI_RESTORE_STATE_AND_DEF_CFA x28, FRAME_SIZE_SAVE_REFS_AND_ARGS 1866 fmov d0, x0 1867 mov x4, FRAME_SIZE_SAVE_REFS_AND_ARGS 1868 bl art_quick_method_exit_hook 1869 b .Lcall_method_exit_hook_done 1870 1871.Lexception_in_native: 1872 // Move to x1 then sp to please assembler. 1873 ldr x1, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 1874 add sp, x1, #-1 // Remove the GenericJNI tag. 1875 bl art_deliver_pending_exception 1876END art_quick_generic_jni_trampoline 1877 1878ENTRY art_deliver_pending_exception 1879 # This will create a new save-all frame, required by the runtime. 1880 DELIVER_PENDING_EXCEPTION 1881END art_deliver_pending_exception 1882 1883/* 1884 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 1885 * of a quick call: 1886 * x0 = method being called/to bridge to. 1887 * x1..x7, d0..d7 = arguments to that method. 1888 */ 1889ENTRY art_quick_to_interpreter_bridge 1890 SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. 1891 1892 // x0 will contain mirror::ArtMethod* method. 1893 mov x1, xSELF // How to get Thread::Current() ??? 1894 mov x2, sp 1895 1896 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 1897 // mirror::ArtMethod** sp) 1898 bl artQuickToInterpreterBridge 1899 1900 RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. 1901 REFRESH_MARKING_REGISTER 1902 1903 fmov d0, x0 1904 1905 RETURN_OR_DELIVER_PENDING_EXCEPTION 1906END art_quick_to_interpreter_bridge 1907 1908/* 1909 * Called to attempt to execute an obsolete method. 1910 */ 1911ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 1912 1913 /* 1914 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 1915 * will long jump to the upcall with a special exception of -1. 1916 */ 1917 .extern artDeoptimizeFromCompiledCode 1918ENTRY art_quick_deoptimize_from_compiled_code 1919 SETUP_SAVE_EVERYTHING_FRAME 1920 mov x1, xSELF // Pass thread. 1921 bl artDeoptimizeFromCompiledCode // (DeoptimizationKind, Thread*) 1922 bl art_quick_do_long_jump // (Context*) 1923 brk 0 // Unreached 1924END art_quick_deoptimize_from_compiled_code 1925 1926 1927 /* 1928 * String's indexOf. 1929 * 1930 * TODO: Not very optimized. 1931 * On entry: 1932 * x0: string object (known non-null) 1933 * w1: char to match (known <= 0xFFFF) 1934 * w2: Starting offset in string data 1935 */ 1936ENTRY art_quick_indexof 1937#if (STRING_COMPRESSION_FEATURE) 1938 ldr w4, [x0, #MIRROR_STRING_COUNT_OFFSET] 1939#else 1940 ldr w3, [x0, #MIRROR_STRING_COUNT_OFFSET] 1941#endif 1942 add x0, x0, #MIRROR_STRING_VALUE_OFFSET 1943#if (STRING_COMPRESSION_FEATURE) 1944 /* w4 holds count (with flag) and w3 holds actual length */ 1945 lsr w3, w4, #1 1946#endif 1947 /* Clamp start to [0..count] */ 1948 cmp w2, #0 1949 csel w2, wzr, w2, lt 1950 cmp w2, w3 1951 csel w2, w3, w2, gt 1952 1953 /* Save a copy to compute result */ 1954 mov x5, x0 1955 1956#if (STRING_COMPRESSION_FEATURE) 1957 tbz w4, #0, .Lstring_indexof_compressed 1958#endif 1959 /* Build pointer to start of data to compare and pre-bias */ 1960 add x0, x0, x2, lsl #1 1961 sub x0, x0, #2 1962 /* Compute iteration count */ 1963 sub w2, w3, w2 1964 1965 /* 1966 * At this point we have: 1967 * x0: start of the data to test 1968 * w1: char to compare 1969 * w2: iteration count 1970 * x5: original start of string data 1971 */ 1972 1973 subs w2, w2, #4 1974 b.lt .Lindexof_remainder 1975 1976.Lindexof_loop4: 1977 ldrh w6, [x0, #2]! 1978 ldrh w7, [x0, #2]! 1979 ldrh wIP0, [x0, #2]! 1980 ldrh wIP1, [x0, #2]! 1981 cmp w6, w1 1982 b.eq .Lmatch_0 1983 cmp w7, w1 1984 b.eq .Lmatch_1 1985 cmp wIP0, w1 1986 b.eq .Lmatch_2 1987 cmp wIP1, w1 1988 b.eq .Lmatch_3 1989 subs w2, w2, #4 1990 b.ge .Lindexof_loop4 1991 1992.Lindexof_remainder: 1993 adds w2, w2, #4 1994 b.eq .Lindexof_nomatch 1995 1996.Lindexof_loop1: 1997 ldrh w6, [x0, #2]! 1998 cmp w6, w1 1999 b.eq .Lmatch_3 2000 subs w2, w2, #1 2001 b.ne .Lindexof_loop1 2002 2003.Lindexof_nomatch: 2004 mov x0, #-1 2005 ret 2006 2007.Lmatch_0: 2008 sub x0, x0, #6 2009 sub x0, x0, x5 2010 asr x0, x0, #1 2011 ret 2012.Lmatch_1: 2013 sub x0, x0, #4 2014 sub x0, x0, x5 2015 asr x0, x0, #1 2016 ret 2017.Lmatch_2: 2018 sub x0, x0, #2 2019 sub x0, x0, x5 2020 asr x0, x0, #1 2021 ret 2022.Lmatch_3: 2023 sub x0, x0, x5 2024 asr x0, x0, #1 2025 ret 2026#if (STRING_COMPRESSION_FEATURE) 2027 /* 2028 * Comparing compressed string character-per-character with 2029 * input character 2030 */ 2031.Lstring_indexof_compressed: 2032 add x0, x0, x2 2033 sub x0, x0, #1 2034 sub w2, w3, w2 2035.Lstring_indexof_compressed_loop: 2036 subs w2, w2, #1 2037 b.lt .Lindexof_nomatch 2038 ldrb w6, [x0, #1]! 2039 cmp w6, w1 2040 b.eq .Lstring_indexof_compressed_matched 2041 b .Lstring_indexof_compressed_loop 2042.Lstring_indexof_compressed_matched: 2043 sub x0, x0, x5 2044 ret 2045#endif 2046END art_quick_indexof 2047 2048 .extern artStringBuilderAppend 2049ENTRY art_quick_string_builder_append 2050 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 2051 add x1, sp, #(FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__) // pass args 2052 mov x2, xSELF // pass Thread::Current 2053 bl artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) 2054 RESTORE_SAVE_REFS_ONLY_FRAME 2055 REFRESH_MARKING_REGISTER 2056 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 2057END art_quick_string_builder_append 2058 2059 /* 2060 * Create a function `name` calling the ReadBarrier::Mark routine, 2061 * getting its argument and returning its result through W register 2062 * `wreg` (corresponding to X register `xreg`), saving and restoring 2063 * all caller-save registers. 2064 * 2065 * The generated function follows a non-standard runtime calling convention: 2066 * - register `reg` (which may be different from W0) is used to pass the (sole) argument, 2067 * - register `reg` (which may be different from W0) is used to return the result, 2068 * - all other registers are callee-save (the values they hold are preserved). 2069 */ 2070.macro READ_BARRIER_MARK_REG name, wreg, xreg 2071ENTRY \name 2072 // Reference is null, no work to do at all. 2073 cbz \wreg, .Lret_rb_\name 2074 // Use wIP0 as temp and check the mark bit of the reference. wIP0 is not used by the compiler. 2075 ldr wIP0, [\xreg, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2076 tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lnot_marked_rb_\name 2077.Lret_rb_\name: 2078 ret 2079.Lnot_marked_rb_\name: 2080 // Check if the top two bits are one, if this is the case it is a forwarding address. 2081 tst wIP0, wIP0, lsl #1 2082 bmi .Lret_forwarding_address\name 2083.Lslow_rb_\name: 2084 /* 2085 * Allocate 44 stack slots * 8 = 352 bytes: 2086 * - 19 slots for core registers X0-15, X17, X19, LR 2087 * - 1 slot padding 2088 * - 24 slots for floating-point registers D0-D7 and D16-D31 2089 */ 2090 // We must not clobber IP1 since code emitted for HLoadClass and HLoadString 2091 // relies on IP1 being preserved. 2092 // Save all potentially live caller-save core registers. 2093 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 352 2094 SAVE_TWO_REGS x2, x3, 16 2095 SAVE_TWO_REGS x4, x5, 32 2096 SAVE_TWO_REGS x6, x7, 48 2097 SAVE_TWO_REGS x8, x9, 64 2098 SAVE_TWO_REGS x10, x11, 80 2099 SAVE_TWO_REGS x12, x13, 96 2100 SAVE_TWO_REGS x14, x15, 112 2101 SAVE_TWO_REGS x17, x19, 128 // Skip x16, i.e. IP0, and x18, the platform register. 2102 SAVE_REG xLR, 144 // Save also return address. 2103 // Save all potentially live caller-save floating-point registers. 2104 stp d0, d1, [sp, #160] 2105 stp d2, d3, [sp, #176] 2106 stp d4, d5, [sp, #192] 2107 stp d6, d7, [sp, #208] 2108 stp d16, d17, [sp, #224] 2109 stp d18, d19, [sp, #240] 2110 stp d20, d21, [sp, #256] 2111 stp d22, d23, [sp, #272] 2112 stp d24, d25, [sp, #288] 2113 stp d26, d27, [sp, #304] 2114 stp d28, d29, [sp, #320] 2115 stp d30, d31, [sp, #336] 2116 2117 .ifnc \wreg, w0 2118 mov w0, \wreg // Pass arg1 - obj from `wreg` 2119 .endif 2120 bl artReadBarrierMark // artReadBarrierMark(obj) 2121 .ifnc \wreg, w0 2122 mov \wreg, w0 // Return result into `wreg` 2123 .endif 2124 2125 // Restore core regs, except `xreg`, as `wreg` is used to return the 2126 // result of this function (simply remove it from the stack instead). 2127 POP_REGS_NE x0, x1, 0, \xreg 2128 POP_REGS_NE x2, x3, 16, \xreg 2129 POP_REGS_NE x4, x5, 32, \xreg 2130 POP_REGS_NE x6, x7, 48, \xreg 2131 POP_REGS_NE x8, x9, 64, \xreg 2132 POP_REGS_NE x10, x11, 80, \xreg 2133 POP_REGS_NE x12, x13, 96, \xreg 2134 POP_REGS_NE x14, x15, 112, \xreg 2135 POP_REGS_NE x17, x19, 128, \xreg 2136 POP_REG_NE xLR, 144, \xreg // Restore also return address. 2137 // Restore floating-point registers. 2138 ldp d0, d1, [sp, #160] 2139 ldp d2, d3, [sp, #176] 2140 ldp d4, d5, [sp, #192] 2141 ldp d6, d7, [sp, #208] 2142 ldp d16, d17, [sp, #224] 2143 ldp d18, d19, [sp, #240] 2144 ldp d20, d21, [sp, #256] 2145 ldp d22, d23, [sp, #272] 2146 ldp d24, d25, [sp, #288] 2147 ldp d26, d27, [sp, #304] 2148 ldp d28, d29, [sp, #320] 2149 ldp d30, d31, [sp, #336] 2150 // Remove frame and return. 2151 DECREASE_FRAME 352 2152 ret 2153.Lret_forwarding_address\name: 2154 // Shift left by the forwarding address shift. This clears out the state bits since they are 2155 // in the top 2 bits of the lock word. 2156 lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2157 ret 2158END \name 2159.endm 2160 2161READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, w0, x0 2162READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, w1, x1 2163READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, w2, x2 2164READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, w3, x3 2165READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, w4, x4 2166READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, w5, x5 2167READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, w6, x6 2168READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, w7, x7 2169READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, w8, x8 2170READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, w9, x9 2171READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, w10, x10 2172READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, w11, x11 2173READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, w12, x12 2174READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, w13, x13 2175READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, w14, x14 2176READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, w15, x15 2177// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, w16, x16 ip0 is blocked 2178READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, w17, x17 2179// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, w18, x18 x18 is blocked 2180READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, w19, x19 2181READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, w20, x20 2182READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, w21, x21 2183READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, w22, x22 2184READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, w23, x23 2185READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, w24, x24 2186READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, w25, x25 2187READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, w26, x26 2188READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, w27, x27 2189READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, w28, x28 2190READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, w29, x29 2191 2192 2193.macro SELECT_X_OR_W_FOR_MACRO macro_to_use, x, w, xreg 2194 .if \xreg 2195 \macro_to_use \x 2196 .else 2197 \macro_to_use \w 2198 .endif 2199.endm 2200 2201.macro FOR_REGISTERS macro_for_register, macro_for_reserved_register, xreg 2202 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x0, w0, \xreg 2203 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x1, w1, \xreg 2204 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x2, w2, \xreg 2205 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x3, w3, \xreg 2206 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x4, w4, \xreg 2207 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x5, w5, \xreg 2208 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x6, w6, \xreg 2209 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x7, w7, \xreg 2210 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x8, w8, \xreg 2211 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x9, w9, \xreg 2212 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x10, w10, \xreg 2213 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x11, w11, \xreg 2214 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x12, w12, \xreg 2215 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x13, w13, \xreg 2216 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x14, w14, \xreg 2217 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x15, w15, \xreg 2218 \macro_for_reserved_register // IP0 is reserved 2219 \macro_for_reserved_register // IP1 is reserved 2220 \macro_for_reserved_register // x18 is reserved 2221 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x19, w19, \xreg 2222 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x20, w20, \xreg 2223 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x21, w21, \xreg 2224 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x22, w22, \xreg 2225 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x23, w23, \xreg 2226 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x24, w24, \xreg 2227 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x25, w25, \xreg 2228 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x26, w26, \xreg 2229 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x27, w27, \xreg 2230 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x28, w28, \xreg 2231 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x29, w29, \xreg 2232 \macro_for_reserved_register // lr is reserved 2233 \macro_for_reserved_register // sp is reserved 2234.endm 2235 2236.macro FOR_XREGISTERS macro_for_register, macro_for_reserved_register 2237 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 1 2238.endm 2239 2240.macro FOR_WREGISTERS macro_for_register, macro_for_reserved_register 2241 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 0 2242.endm 2243 2244.macro BRK0_BRK0 2245 brk 0 2246 brk 0 2247.endm 2248 2249#if BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET != BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET 2250#error "Array and field introspection code sharing requires same LDR offset." 2251#endif 2252.macro INTROSPECTION_ARRAY_LOAD index_reg 2253 ldr wIP0, [xIP0, \index_reg, lsl #2] 2254 b art_quick_read_barrier_mark_introspection 2255.endm 2256 2257.macro MOV_WIP0_TO_WREG_AND_BL_LR reg 2258 mov \reg, wIP0 2259 br lr // Do not use RET as we do not enter the entrypoint with "BL". 2260.endm 2261 2262.macro READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH ldr_offset 2263 /* 2264 * Allocate 42 stack slots * 8 = 336 bytes: 2265 * - 18 slots for core registers X0-15, X19, LR 2266 * - 24 slots for floating-point registers D0-D7 and D16-D31 2267 */ 2268 // Save all potentially live caller-save core registers. 2269 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 336 2270 SAVE_TWO_REGS x2, x3, 16 2271 SAVE_TWO_REGS x4, x5, 32 2272 SAVE_TWO_REGS x6, x7, 48 2273 SAVE_TWO_REGS x8, x9, 64 2274 SAVE_TWO_REGS x10, x11, 80 2275 SAVE_TWO_REGS x12, x13, 96 2276 SAVE_TWO_REGS x14, x15, 112 2277 // Skip x16, x17, i.e. IP0, IP1, and x18, the platform register. 2278 SAVE_TWO_REGS x19, xLR, 128 // Save return address. 2279 // Save all potentially live caller-save floating-point registers. 2280 stp d0, d1, [sp, #144] 2281 stp d2, d3, [sp, #160] 2282 stp d4, d5, [sp, #176] 2283 stp d6, d7, [sp, #192] 2284 stp d16, d17, [sp, #208] 2285 stp d18, d19, [sp, #224] 2286 stp d20, d21, [sp, #240] 2287 stp d22, d23, [sp, #256] 2288 stp d24, d25, [sp, #272] 2289 stp d26, d27, [sp, #288] 2290 stp d28, d29, [sp, #304] 2291 stp d30, d31, [sp, #320] 2292 2293 mov x0, xIP0 2294 bl artReadBarrierMark // artReadBarrierMark(obj) 2295 mov xIP0, x0 2296 2297 // Restore core regs, except x0 and x1 as the return register switch case 2298 // address calculation is smoother with an extra register. 2299 RESTORE_TWO_REGS x2, x3, 16 2300 RESTORE_TWO_REGS x4, x5, 32 2301 RESTORE_TWO_REGS x6, x7, 48 2302 RESTORE_TWO_REGS x8, x9, 64 2303 RESTORE_TWO_REGS x10, x11, 80 2304 RESTORE_TWO_REGS x12, x13, 96 2305 RESTORE_TWO_REGS x14, x15, 112 2306 // Skip x16, x17, i.e. IP0, IP1, and x18, the platform register. 2307 RESTORE_TWO_REGS x19, xLR, 128 // Restore return address. 2308 // Restore caller-save floating-point registers. 2309 ldp d0, d1, [sp, #144] 2310 ldp d2, d3, [sp, #160] 2311 ldp d4, d5, [sp, #176] 2312 ldp d6, d7, [sp, #192] 2313 ldp d16, d17, [sp, #208] 2314 ldp d18, d19, [sp, #224] 2315 ldp d20, d21, [sp, #240] 2316 ldp d22, d23, [sp, #256] 2317 ldp d24, d25, [sp, #272] 2318 ldp d26, d27, [sp, #288] 2319 ldp d28, d29, [sp, #304] 2320 ldp d30, d31, [sp, #320] 2321 2322 ldr x0, [lr, #\ldr_offset] // Load the instruction. 2323 adr xIP1, .Lmark_introspection_return_switch 2324 bfi xIP1, x0, #3, #5 // Calculate switch case address. 2325 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 336 2326 br xIP1 2327.endm 2328 2329 /* 2330 * Use introspection to load a reference from the same address as the LDR 2331 * instruction in generated code would load (unless loaded by the thunk, 2332 * see below), call ReadBarrier::Mark() with that reference if needed 2333 * and return it in the same register as the LDR instruction would load. 2334 * 2335 * The entrypoint is called through a thunk that differs across load kinds. 2336 * For field and array loads the LDR instruction in generated code follows 2337 * the branch to the thunk, i.e. the LDR is at [LR, #-4], and the thunk 2338 * knows the holder and performs the gray bit check, returning to the LDR 2339 * instruction if the object is not gray, so this entrypoint no longer 2340 * needs to know anything about the holder. For GC root loads, the LDR 2341 * instruction in generated code precedes the branch to the thunk (i.e. 2342 * the LDR is at [LR, #-8]) and the thunk does not do the gray bit check. 2343 * 2344 * For field accesses and array loads with a constant index the thunk loads 2345 * the reference into IP0 using introspection and calls the main entrypoint, 2346 * art_quick_read_barrier_mark_introspection. With heap poisoning enabled, 2347 * the passed reference is poisoned. 2348 * 2349 * For array accesses with non-constant index, the thunk inserts the bits 2350 * 16-21 of the LDR instruction to the entrypoint address, effectively 2351 * calculating a switch case label based on the index register (bits 16-20) 2352 * and adding an extra offset (bit 21 is set) to differentiate from the 2353 * main entrypoint, then moves the base register to IP0 and jumps to the 2354 * switch case. Therefore we need to align the main entrypoint to 512 bytes, 2355 * accounting for a 256-byte offset followed by 32 array entrypoints 2356 * starting at art_quick_read_barrier_mark_introspection_arrays, each 2357 * containing an LDR (register) and a branch to the main entrypoint. 2358 * 2359 * For GC root accesses we cannot use the main entrypoint because of the 2360 * different offset where the LDR instruction in generated code is located. 2361 * (And even with heap poisoning enabled, GC roots are not poisoned.) 2362 * To re-use the same entrypoint pointer in generated code, we make sure 2363 * that the gc root entrypoint (a copy of the entrypoint with a different 2364 * offset for introspection loads) is located at a known offset (768 bytes, 2365 * or BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET) from the main 2366 * entrypoint and the GC root thunk adjusts the entrypoint pointer, moves 2367 * the root register to IP0 and jumps to the customized entrypoint, 2368 * art_quick_read_barrier_mark_introspection_gc_roots. The thunk also 2369 * performs all the fast-path checks, so we need just the slow path. 2370 * The UnsafeCASObject intrinsic is also using the GC root entrypoint with 2371 * MOV instead of LDR, the destination register is in the same bits. 2372 * 2373 * The code structure is 2374 * art_quick_read_barrier_mark_introspection: 2375 * Up to 256 bytes for the main entrypoint code. 2376 * Padding to 256 bytes if needed. 2377 * art_quick_read_barrier_mark_introspection_arrays: 2378 * Exactly 256 bytes for array load switch cases (32x2 instructions). 2379 * .Lmark_introspection_return_switch: 2380 * Exactly 256 bytes for return switch cases (32x2 instructions). 2381 * art_quick_read_barrier_mark_introspection_gc_roots: 2382 * GC root entrypoint code. 2383 */ 2384ENTRY_ALIGNED art_quick_read_barrier_mark_introspection, 512 2385 // At this point, IP0 contains the reference, IP1 can be freely used. 2386 // For heap poisoning, the reference is poisoned, so unpoison it first. 2387 UNPOISON_HEAP_REF wIP0 2388 // If reference is null, just return it in the right register. 2389 cbz wIP0, .Lmark_introspection_return 2390 // Use wIP1 as temp and check the mark bit of the reference. 2391 ldr wIP1, [xIP0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2392 tbz wIP1, #LOCK_WORD_MARK_BIT_SHIFT, .Lmark_introspection_unmarked 2393.Lmark_introspection_return: 2394 // Without an extra register for the return switch case address calculation, 2395 // we exploit the high word of the xIP0 to temporarily store the ref_reg*8, 2396 // so the return switch below must move wIP0 instead of xIP0 to the register. 2397 ldr wIP1, [lr, #BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET] // Load the instruction. 2398 bfi xIP0, xIP1, #(32 + 3), #5 // Extract ref_reg*8 to high word in xIP0. 2399 adr xIP1, .Lmark_introspection_return_switch 2400 bfxil xIP1, xIP0, #32, #8 // Calculate return switch case address. 2401 br xIP1 2402.Lmark_introspection_unmarked: 2403 // Check if the top two bits are one, if this is the case it is a forwarding address. 2404 tst wIP1, wIP1, lsl #1 2405 bmi .Lmark_introspection_forwarding_address 2406 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET 2407 2408.Lmark_introspection_forwarding_address: 2409 // Shift left by the forwarding address shift. This clears out the state bits since they are 2410 // in the top 2 bits of the lock word. 2411 lsl wIP0, wIP1, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2412 b .Lmark_introspection_return 2413 2414 // We're very close to the alloted 256B for the entrypoint code before the 2415 // array switch cases. Should we go a little bit over the limit, we can 2416 // move some code after the array switch cases and return switch cases. 2417 .balign 256 2418 .hidden art_quick_read_barrier_mark_introspection_arrays 2419 .global art_quick_read_barrier_mark_introspection_arrays 2420art_quick_read_barrier_mark_introspection_arrays: 2421 FOR_XREGISTERS INTROSPECTION_ARRAY_LOAD, BRK0_BRK0 2422.Lmark_introspection_return_switch: 2423 FOR_WREGISTERS MOV_WIP0_TO_WREG_AND_BL_LR, BRK0_BRK0 2424 .hidden art_quick_read_barrier_mark_introspection_gc_roots 2425 .global art_quick_read_barrier_mark_introspection_gc_roots 2426art_quick_read_barrier_mark_introspection_gc_roots: 2427 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_OFFSET 2428END art_quick_read_barrier_mark_introspection 2429 2430.extern artInvokePolymorphic 2431ENTRY art_quick_invoke_polymorphic 2432 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2433 mov x0, x1 // x0 := receiver 2434 mov x1, xSELF // x1 := Thread::Current() 2435 mov x2, sp // x2 := SP 2436 bl artInvokePolymorphic // artInvokePolymorphic(receiver, thread, save_area) 2437 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2438 REFRESH_MARKING_REGISTER 2439 fmov d0, x0 // Result is in x0. Copy to floating return register. 2440 RETURN_OR_DELIVER_PENDING_EXCEPTION 2441END art_quick_invoke_polymorphic 2442 2443 /* 2444 * Slow path for MethodHandle.invokeExact intrinsic. 2445 * That intrinsic has a custom calling convention: the argument allocation doesn't start from 2446 * the receiver (MethodHandle) object, but from the argument following it. That's done to match 2447 * expectation of the underlying method when MethodHandle targets a method. That also affects 2448 * the way arguments are spilled onto the stack. 2449 */ 2450.extern artInvokePolymorphicWithHiddenReceiver 2451ENTRY art_quick_invoke_polymorphic_with_hidden_receiver 2452 // On entry: w0 := receiver 2453 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2454 mov x1, xSELF // x1 := Thread::Current() 2455 mov x2, sp // x2 := SP 2456 bl artInvokePolymorphicWithHiddenReceiver // invoke with (receiver, thread, save_area) 2457 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2458 REFRESH_MARKING_REGISTER 2459 fmov d0, x0 // Result is in x0. Copy to floating return register. 2460 RETURN_OR_DELIVER_PENDING_EXCEPTION 2461END art_quick_invoke_polymorphic_with_hidden_receiver 2462 2463.extern artInvokeCustom 2464ENTRY art_quick_invoke_custom 2465 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2466 // x0 := call_site_idx 2467 mov x1, xSELF // x1 := Thread::Current() 2468 mov x2, sp // x2 := SP 2469 bl artInvokeCustom // artInvokeCustom(call_site_idx, thread, save_area) 2470 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2471 REFRESH_MARKING_REGISTER 2472 fmov d0, x0 // Copy result to double result register. 2473 RETURN_OR_DELIVER_PENDING_EXCEPTION 2474END art_quick_invoke_custom 2475 2476// x0 contains the class, x8 contains the inline cache. x9-x15 can be used. 2477ENTRY art_quick_update_inline_cache 2478#if (INLINE_CACHE_SIZE != 5) 2479#error "INLINE_CACHE_SIZE not as expected." 2480#endif 2481#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2482 // Don't update the cache if we are marking. 2483 cbnz wMR, .Ldone 2484#endif 2485.Lentry1: 2486 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET] 2487 cmp w9, w0 2488 beq .Ldone 2489 cbnz w9, .Lentry2 2490 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET 2491 ldxr w9, [x10] 2492 cbnz w9, .Lentry1 2493 stxr w9, w0, [x10] 2494 cbz w9, .Ldone 2495 b .Lentry1 2496.Lentry2: 2497 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+4] 2498 cmp w9, w0 2499 beq .Ldone 2500 cbnz w9, .Lentry3 2501 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+4 2502 ldxr w9, [x10] 2503 cbnz w9, .Lentry2 2504 stxr w9, w0, [x10] 2505 cbz w9, .Ldone 2506 b .Lentry2 2507.Lentry3: 2508 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+8] 2509 cmp w9, w0 2510 beq .Ldone 2511 cbnz w9, .Lentry4 2512 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+8 2513 ldxr w9, [x10] 2514 cbnz w9, .Lentry3 2515 stxr w9, w0, [x10] 2516 cbz w9, .Ldone 2517 b .Lentry3 2518.Lentry4: 2519 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+12] 2520 cmp w9, w0 2521 beq .Ldone 2522 cbnz w9, .Lentry5 2523 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+12 2524 ldxr w9, [x10] 2525 cbnz w9, .Lentry4 2526 stxr w9, w0, [x10] 2527 cbz w9, .Ldone 2528 b .Lentry4 2529.Lentry5: 2530 // Unconditionally store, the inline cache is megamorphic. 2531 str w0, [x8, #INLINE_CACHE_CLASSES_OFFSET+16] 2532.Ldone: 2533 ret 2534END art_quick_update_inline_cache 2535 2536// On entry, method is at the bottom of the stack. 2537ENTRY art_quick_compile_optimized 2538 SETUP_SAVE_EVERYTHING_FRAME 2539 ldr x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass ArtMethod 2540 mov x1, xSELF // pass Thread::Current 2541 bl artCompileOptimized // (ArtMethod*, Thread*) 2542 RESTORE_SAVE_EVERYTHING_FRAME 2543 // We don't need to restore the marking register here, as 2544 // artCompileOptimized doesn't allow thread suspension. 2545 ret 2546END art_quick_compile_optimized 2547 2548ENTRY art_quick_record_entry_trace_event 2549 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2550 // xIP0 has the trace buffer pointer. This is loaded on the fast path before 2551 // checking if we need to call this method. This will be still valid here. 2552 cmp xIP1, xIP0 2553 bhs .Lupdate_entry 2554 mov xIP1, #TRACE_BUFFER_SIZE 2555 add xIP1, xIP0, xIP1 2556.Lupdate_entry: 2557 sub xIP1, xIP1, 8 2558 str x0, [xIP1] 2559 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2560 ret 2561END art_quick_record_entry_trace_event 2562 2563ENTRY art_quick_record_exit_trace_event 2564 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2565 // xIP0 has the trace buffer pointer. This is loaded on the fast path before 2566 // checking if we need to call this method. This will be still valid here. 2567 cmp xIP1, xIP0 2568 bhs .Lupdate_entry_exit 2569 mov xIP1, #TRACE_BUFFER_SIZE 2570 add xIP1, xIP0, xIP1 2571.Lupdate_entry_exit: 2572 sub xIP1, xIP1, 8 2573 mov xIP0, #1 2574 str xIP0, [xIP1] 2575 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2576 ret 2577END art_quick_record_exit_trace_event 2578 2579 .extern artMethodEntryHook 2580ENTRY art_quick_method_entry_hook 2581 SETUP_SAVE_EVERYTHING_FRAME 2582 2583 ldr x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass ArtMethod* 2584 mov x1, xSELF // pass Thread::Current 2585 mov x2, sp // pass SP 2586 bl artMethodEntryHook // (ArtMethod*, Thread*, SP) 2587 2588 CFI_REMEMBER_STATE 2589 cbnz x0, .Lentryhook_deopt 2590 2591 // Normal return. 2592 RESTORE_SAVE_EVERYTHING_FRAME // Note: will restore xSELF 2593 REFRESH_MARKING_REGISTER 2594 ret 2595 2596.Lentryhook_deopt: 2597 // Deoptimize. 2598 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 2599 bl art_quick_do_long_jump // (Context*) 2600 brk 0 // Unreached 2601END art_quick_method_entry_hook 2602 2603 .extern artMethodExitHook 2604ENTRY art_quick_method_exit_hook 2605 SETUP_SAVE_EVERYTHING_FRAME 2606 2607 // frame_size is passed from JITed code in x4 2608 add x3, sp, #16 // floating-point result ptr in kSaveEverything frame 2609 add x2, sp, #272 // integer result ptr in kSaveEverything frame 2610 add x1, sp, #FRAME_SIZE_SAVE_EVERYTHING // ArtMethod** 2611 mov x0, xSELF // Thread::Current 2612 bl artMethodExitHook // (Thread*, ArtMethod**, gpr_res*, fpr_res*, 2613 // frame_size) 2614 2615 CFI_REMEMBER_STATE 2616 cbnz x0, .Lexithook_deopt_or_exception 2617 2618 // Normal return. 2619 RESTORE_SAVE_EVERYTHING_FRAME 2620 REFRESH_MARKING_REGISTER 2621 ret 2622 2623.Lexithook_deopt_or_exception: 2624 // Deoptimize or exception thrown. 2625 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 2626 bl art_quick_do_long_jump // (Context*) 2627 brk 0 // Unreached 2628END art_quick_method_exit_hook 2629