1 //===- subzero/src/IceAssemblerX8664Impl.h - base x86 assembler -*- C++ -*-=//
2 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
3 // for details. All rights reserved. Use of this source code is governed by a
4 // BSD-style license that can be found in the LICENSE file.
5 //
6 // Modified by the Subzero authors.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // The Subzero Code Generator
11 //
12 // This file is distributed under the University of Illinois Open Source
13 // License. See LICENSE.TXT for details.
14 //
15 //===----------------------------------------------------------------------===//
16 //
17 /// \file
18 /// \brief Implements the AssemblerX8664 class.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "IceAssemblerX8664.h"
23
24 #include "IceCfg.h"
25 #include "IceCfgNode.h"
26 #include "IceOperand.h"
27 #include "IceTargetLoweringX8664.h"
28
29 namespace Ice {
30 namespace X8664 {
31
AsmAddress(const Variable * Var,const TargetX8664 * Target)32 AsmAddress::AsmAddress(const Variable *Var, const TargetX8664 *Target) {
33 if (Var->hasReg())
34 llvm::report_fatal_error("Stack Variable has a register assigned");
35 if (Var->mustHaveReg()) {
36 llvm::report_fatal_error("Infinite-weight Variable (" + Var->getName() +
37 ") has no register assigned - function " +
38 Target->getFunc()->getFunctionName());
39 }
40 int32_t Offset = Var->getStackOffset();
41 auto BaseRegNum = Var->getBaseRegNum();
42 if (Var->getBaseRegNum().hasNoValue()) {
43 // If the stack pointer needs alignment, we must use the frame pointer for
44 // arguments. For locals, getFrameOrStackReg will return the stack pointer
45 // in this case.
46 if (Target->needsStackPointerAlignment() && Var->getIsArg()) {
47 assert(Target->hasFramePointer());
48 BaseRegNum = Target->getFrameReg();
49 } else {
50 BaseRegNum = Target->getFrameOrStackReg();
51 }
52 }
53 SetBase(RegX8664::getEncodedGPR(BaseRegNum), Offset, AssemblerFixup::NoFixup);
54 }
55
AsmAddress(const X86OperandMem * Mem,Ice::Assembler * Asm,const Ice::TargetLowering * Target)56 AsmAddress::AsmAddress(const X86OperandMem *Mem, Ice::Assembler *Asm,
57 const Ice::TargetLowering *Target) {
58 int32_t Disp = 0;
59 if (Mem->getBase() && Mem->getBase()->isRematerializable()) {
60 Disp += Mem->getBase()->getRematerializableOffset(Target);
61 }
62 // The index should never be rematerializable. But if we ever allow it, then
63 // we should make sure the rematerialization offset is shifted by the Shift
64 // value.
65 assert(!Mem->getIndex() || !Mem->getIndex()->isRematerializable());
66
67 AssemblerFixup *Fixup = nullptr;
68 // Determine the offset (is it relocatable?)
69 if (Mem->getOffset() != nullptr) {
70 if (const auto *CI = llvm::dyn_cast<ConstantInteger32>(Mem->getOffset())) {
71 Disp += static_cast<int32_t>(CI->getValue());
72 } else if (const auto *CR =
73 llvm::dyn_cast<ConstantRelocatable>(Mem->getOffset())) {
74 const auto FixupKind =
75 (Mem->getBase() != nullptr || Mem->getIndex() != nullptr) ? FK_Abs
76 : FK_PcRel;
77 const RelocOffsetT DispAdjustment = FixupKind == FK_PcRel ? 4 : 0;
78 Fixup = Asm->createFixup(FixupKind, CR);
79 Fixup->set_addend(-DispAdjustment);
80 Disp = CR->getOffset();
81 } else {
82 llvm_unreachable("Unexpected offset type");
83 }
84 }
85
86 // Now convert to the various possible forms.
87 if (Mem->getBase() && Mem->getIndex()) {
88 SetBaseIndex(RegX8664::getEncodedGPR(Mem->getBase()->getRegNum()),
89 RegX8664::getEncodedGPR(Mem->getIndex()->getRegNum()),
90 ScaleFactor(Mem->getShift()), Disp, Fixup);
91 } else if (Mem->getBase()) {
92 SetBase(RegX8664::getEncodedGPR(Mem->getBase()->getRegNum()), Disp, Fixup);
93 } else if (Mem->getIndex()) {
94 SetIndex(RegX8664::getEncodedGPR(Mem->getIndex()->getRegNum()),
95 ScaleFactor(Mem->getShift()), Disp, Fixup);
96 } else if (Fixup == nullptr) {
97 SetAbsolute(Disp);
98 } else {
99 SetRipRelative(Disp, Fixup);
100 }
101 }
102
~AssemblerX8664()103 AssemblerX8664::~AssemblerX8664() {
104 if (BuildDefs::asserts()) {
105 for (const Label *Label : CfgNodeLabels) {
106 Label->finalCheck();
107 }
108 for (const Label *Label : LocalLabels) {
109 Label->finalCheck();
110 }
111 }
112 }
113
alignFunction()114 void AssemblerX8664::alignFunction() {
115 const SizeT Align = 1 << getBundleAlignLog2Bytes();
116 SizeT BytesNeeded = Utils::OffsetToAlignment(Buffer.getPosition(), Align);
117 constexpr SizeT HltSize = 1;
118 while (BytesNeeded > 0) {
119 hlt();
120 BytesNeeded -= HltSize;
121 }
122 }
123
getOrCreateLabel(SizeT Number,LabelVector & Labels)124 AssemblerX8664::Label *AssemblerX8664::getOrCreateLabel(SizeT Number,
125 LabelVector &Labels) {
126 Label *L = nullptr;
127 if (Number == Labels.size()) {
128 L = new (this->allocate<Label>()) Label();
129 Labels.push_back(L);
130 return L;
131 }
132 if (Number > Labels.size()) {
133 Utils::reserveAndResize(Labels, Number + 1);
134 }
135 L = Labels[Number];
136 if (!L) {
137 L = new (this->allocate<Label>()) Label();
138 Labels[Number] = L;
139 }
140 return L;
141 }
142
getCfgNodeLabel(SizeT NodeNumber)143 Ice::Label *AssemblerX8664::getCfgNodeLabel(SizeT NodeNumber) {
144 assert(NodeNumber < CfgNodeLabels.size());
145 return CfgNodeLabels[NodeNumber];
146 }
147
148 AssemblerX8664::Label *
getOrCreateCfgNodeLabel(SizeT NodeNumber)149 AssemblerX8664::getOrCreateCfgNodeLabel(SizeT NodeNumber) {
150 return getOrCreateLabel(NodeNumber, CfgNodeLabels);
151 }
152
getOrCreateLocalLabel(SizeT Number)153 AssemblerX8664::Label *AssemblerX8664::getOrCreateLocalLabel(SizeT Number) {
154 return getOrCreateLabel(Number, LocalLabels);
155 }
156
bindCfgNodeLabel(const CfgNode * Node)157 void AssemblerX8664::bindCfgNodeLabel(const CfgNode *Node) {
158 assert(!getPreliminary());
159 Label *L = getOrCreateCfgNodeLabel(Node->getIndex());
160 this->bind(L);
161 }
162
bindLocalLabel(SizeT Number)163 void AssemblerX8664::bindLocalLabel(SizeT Number) {
164 Label *L = getOrCreateLocalLabel(Number);
165 if (!getPreliminary())
166 this->bind(L);
167 }
168
call(GPRRegister reg)169 void AssemblerX8664::call(GPRRegister reg) {
170 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
171 emitRexB(RexTypeIrrelevant, reg);
172 emitUint8(0xFF);
173 emitRegisterOperand(2, gprEncoding(reg));
174 }
175
call(const AsmAddress & address)176 void AssemblerX8664::call(const AsmAddress &address) {
177 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
178 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
179 emitUint8(0xFF);
180 emitOperand(2, address);
181 }
182
call(const ConstantRelocatable * label)183 void AssemblerX8664::call(const ConstantRelocatable *label) {
184 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
185 intptr_t call_start = Buffer.getPosition();
186 emitUint8(0xE8);
187 auto *Fixup = this->createFixup(FK_PcRel, label);
188 Fixup->set_addend(-4);
189 emitFixup(Fixup);
190 emitInt32(0);
191 assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
192 (void)call_start;
193 }
194
call(const Immediate & abs_address)195 void AssemblerX8664::call(const Immediate &abs_address) {
196 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
197 intptr_t call_start = Buffer.getPosition();
198 emitUint8(0xE8);
199 auto *Fixup = this->createFixup(FK_PcRel, AssemblerFixup::NullSymbol);
200 Fixup->set_addend(abs_address.value() - 4);
201 emitFixup(Fixup);
202 emitInt32(0);
203 assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
204 (void)call_start;
205 }
206
pushl(GPRRegister reg)207 void AssemblerX8664::pushl(GPRRegister reg) {
208 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
209 emitRexB(RexTypeIrrelevant, reg);
210 emitUint8(0x50 + gprEncoding(reg));
211 }
212
pushl(const Immediate & Imm)213 void AssemblerX8664::pushl(const Immediate &Imm) {
214 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
215 emitUint8(0x68);
216 emitInt32(Imm.value());
217 }
218
pushl(const ConstantRelocatable * Label)219 void AssemblerX8664::pushl(const ConstantRelocatable *Label) {
220 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
221 emitUint8(0x68);
222 emitFixup(this->createFixup(FK_Abs, Label));
223 // In x86-32, the emitted value is an addend to the relocation. Therefore, we
224 // must emit a 0 (because we're pushing an absolute relocation.)
225 // In x86-64, the emitted value does not matter (the addend lives in the
226 // relocation record as an extra field.)
227 emitInt32(0);
228 }
229
popl(GPRRegister reg)230 void AssemblerX8664::popl(GPRRegister reg) {
231 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
232 // Any type that would not force a REX prefix to be emitted can be provided
233 // here.
234 emitRexB(RexTypeIrrelevant, reg);
235 emitUint8(0x58 + gprEncoding(reg));
236 }
237
popl(const AsmAddress & address)238 void AssemblerX8664::popl(const AsmAddress &address) {
239 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
240 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
241 emitUint8(0x8F);
242 emitOperand(0, address);
243 }
244
setcc(BrCond condition,ByteRegister dst)245 void AssemblerX8664::setcc(BrCond condition, ByteRegister dst) {
246 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
247 emitRexB(IceType_i8, dst);
248 emitUint8(0x0F);
249 emitUint8(0x90 + condition);
250 emitUint8(0xC0 + gprEncoding(dst));
251 }
252
setcc(BrCond condition,const AsmAddress & address)253 void AssemblerX8664::setcc(BrCond condition, const AsmAddress &address) {
254 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
255 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
256 emitUint8(0x0F);
257 emitUint8(0x90 + condition);
258 emitOperand(0, address);
259 }
260
mov(Type Ty,GPRRegister dst,const Immediate & imm)261 void AssemblerX8664::mov(Type Ty, GPRRegister dst, const Immediate &imm) {
262 assert(Ty != IceType_i64 && "i64 not supported yet.");
263 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
264 if (Ty == IceType_i16)
265 emitOperandSizeOverride();
266 emitRexB(Ty, dst);
267 if (isByteSizedType(Ty)) {
268 emitUint8(0xB0 + gprEncoding(dst));
269 emitUint8(imm.value() & 0xFF);
270 } else {
271 // TODO(jpp): When removing the assertion above ensure that in x86-64 we
272 // emit a 64-bit immediate.
273 emitUint8(0xB8 + gprEncoding(dst));
274 emitImmediate(Ty, imm);
275 }
276 }
277
mov(Type Ty,GPRRegister dst,GPRRegister src)278 void AssemblerX8664::mov(Type Ty, GPRRegister dst, GPRRegister src) {
279 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
280 if (Ty == IceType_i16)
281 emitOperandSizeOverride();
282 emitRexRB(Ty, src, dst);
283 if (isByteSizedType(Ty)) {
284 emitUint8(0x88);
285 } else {
286 emitUint8(0x89);
287 }
288 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
289 }
290
mov(Type Ty,GPRRegister dst,const AsmAddress & src)291 void AssemblerX8664::mov(Type Ty, GPRRegister dst, const AsmAddress &src) {
292 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
293 if (Ty == IceType_i16)
294 emitOperandSizeOverride();
295 emitRex(Ty, src, dst);
296 if (isByteSizedType(Ty)) {
297 emitUint8(0x8A);
298 } else {
299 emitUint8(0x8B);
300 }
301 emitOperand(gprEncoding(dst), src);
302 }
303
mov(Type Ty,const AsmAddress & dst,GPRRegister src)304 void AssemblerX8664::mov(Type Ty, const AsmAddress &dst, GPRRegister src) {
305 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
306 if (Ty == IceType_i16)
307 emitOperandSizeOverride();
308 emitRex(Ty, dst, src);
309 if (isByteSizedType(Ty)) {
310 emitUint8(0x88);
311 } else {
312 emitUint8(0x89);
313 }
314 emitOperand(gprEncoding(src), dst);
315 }
316
mov(Type Ty,const AsmAddress & dst,const Immediate & imm)317 void AssemblerX8664::mov(Type Ty, const AsmAddress &dst, const Immediate &imm) {
318 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
319 if (Ty == IceType_i16)
320 emitOperandSizeOverride();
321 emitRex(Ty, dst, RexRegIrrelevant);
322 if (isByteSizedType(Ty)) {
323 emitUint8(0xC6);
324 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
325 emitOperand(0, dst, OffsetFromNextInstruction);
326 emitUint8(imm.value() & 0xFF);
327 } else {
328 emitUint8(0xC7);
329 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
330 emitOperand(0, dst, OffsetFromNextInstruction);
331 emitImmediate(Ty, imm);
332 }
333 }
334
movabs(const GPRRegister Dst,uint64_t Imm64)335 void AssemblerX8664::movabs(const GPRRegister Dst, uint64_t Imm64) {
336 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
337 const bool NeedsRexW = (Imm64 & ~0xFFFFFFFFull) != 0;
338 const Type RexType = NeedsRexW ? RexTypeForceRexW : RexTypeIrrelevant;
339 emitRexB(RexType, Dst);
340 emitUint8(0xB8 | gprEncoding(Dst));
341 // When emitting Imm64, we don't have to mask out the upper 32 bits for
342 // emitInt32 will/should only emit a 32-bit constant. In reality, we are
343 // paranoid, so we go ahead an mask the upper bits out anyway.
344 emitInt32(Imm64 & 0xFFFFFFFF);
345 if (NeedsRexW)
346 emitInt32((Imm64 >> 32) & 0xFFFFFFFF);
347 }
348
movzx(Type SrcTy,GPRRegister dst,GPRRegister src)349 void AssemblerX8664::movzx(Type SrcTy, GPRRegister dst, GPRRegister src) {
350 if (SrcTy == IceType_i32) {
351 // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
352 // operand to 64-bit.
353 mov(IceType_i32, dst, src);
354 return;
355 }
356
357 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
358 bool ByteSized = isByteSizedType(SrcTy);
359 assert(ByteSized || SrcTy == IceType_i16);
360 emitRexRB(RexTypeIrrelevant, dst, SrcTy, src);
361 emitUint8(0x0F);
362 emitUint8(ByteSized ? 0xB6 : 0xB7);
363 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
364 }
365
movzx(Type SrcTy,GPRRegister dst,const AsmAddress & src)366 void AssemblerX8664::movzx(Type SrcTy, GPRRegister dst, const AsmAddress &src) {
367 if (SrcTy == IceType_i32) {
368 // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
369 // operand to 64-bit.
370 mov(IceType_i32, dst, src);
371 return;
372 }
373
374 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
375 bool ByteSized = isByteSizedType(SrcTy);
376 assert(ByteSized || SrcTy == IceType_i16);
377 emitRex(SrcTy, src, RexTypeIrrelevant, dst);
378 emitUint8(0x0F);
379 emitUint8(ByteSized ? 0xB6 : 0xB7);
380 emitOperand(gprEncoding(dst), src);
381 }
382
movsx(Type SrcTy,GPRRegister dst,GPRRegister src)383 void AssemblerX8664::movsx(Type SrcTy, GPRRegister dst, GPRRegister src) {
384 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
385 bool ByteSized = isByteSizedType(SrcTy);
386 emitRexRB(RexTypeForceRexW, dst, SrcTy, src);
387 if (ByteSized || SrcTy == IceType_i16) {
388 emitUint8(0x0F);
389 emitUint8(ByteSized ? 0xBE : 0xBF);
390 } else {
391 assert(SrcTy == IceType_i32);
392 emitUint8(0x63);
393 }
394 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
395 }
396
movsx(Type SrcTy,GPRRegister dst,const AsmAddress & src)397 void AssemblerX8664::movsx(Type SrcTy, GPRRegister dst, const AsmAddress &src) {
398 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
399 bool ByteSized = isByteSizedType(SrcTy);
400 emitRex(SrcTy, src, RexTypeForceRexW, dst);
401 if (ByteSized || SrcTy == IceType_i16) {
402 emitUint8(0x0F);
403 emitUint8(ByteSized ? 0xBE : 0xBF);
404 } else {
405 assert(SrcTy == IceType_i32);
406 emitUint8(0x63);
407 }
408 emitOperand(gprEncoding(dst), src);
409 }
410
lea(Type Ty,GPRRegister dst,const AsmAddress & src)411 void AssemblerX8664::lea(Type Ty, GPRRegister dst, const AsmAddress &src) {
412 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
413 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
414 if (Ty == IceType_i16)
415 emitOperandSizeOverride();
416 emitRex(Ty, src, dst);
417 emitUint8(0x8D);
418 emitOperand(gprEncoding(dst), src);
419 }
420
cmov(Type Ty,BrCond cond,GPRRegister dst,GPRRegister src)421 void AssemblerX8664::cmov(Type Ty, BrCond cond, GPRRegister dst,
422 GPRRegister src) {
423 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
424 if (Ty == IceType_i16)
425 emitOperandSizeOverride();
426 else
427 assert(Ty == IceType_i32 || Ty == IceType_i64);
428 emitRexRB(Ty, dst, src);
429 emitUint8(0x0F);
430 emitUint8(0x40 + cond);
431 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
432 }
433
cmov(Type Ty,BrCond cond,GPRRegister dst,const AsmAddress & src)434 void AssemblerX8664::cmov(Type Ty, BrCond cond, GPRRegister dst,
435 const AsmAddress &src) {
436 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
437 if (Ty == IceType_i16)
438 emitOperandSizeOverride();
439 else
440 assert(Ty == IceType_i32 || Ty == IceType_i64);
441 emitRex(Ty, src, dst);
442 emitUint8(0x0F);
443 emitUint8(0x40 + cond);
444 emitOperand(gprEncoding(dst), src);
445 }
446
rep_movsb()447 void AssemblerX8664::rep_movsb() {
448 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
449 emitUint8(0xF3);
450 emitUint8(0xA4);
451 }
452
movss(Type Ty,XmmRegister dst,const AsmAddress & src)453 void AssemblerX8664::movss(Type Ty, XmmRegister dst, const AsmAddress &src) {
454 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
455 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
456 emitRex(RexTypeIrrelevant, src, dst);
457 emitUint8(0x0F);
458 emitUint8(0x10);
459 emitOperand(gprEncoding(dst), src);
460 }
461
movss(Type Ty,const AsmAddress & dst,XmmRegister src)462 void AssemblerX8664::movss(Type Ty, const AsmAddress &dst, XmmRegister src) {
463 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
464 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
465 emitRex(RexTypeIrrelevant, dst, src);
466 emitUint8(0x0F);
467 emitUint8(0x11);
468 emitOperand(gprEncoding(src), dst);
469 }
470
movss(Type Ty,XmmRegister dst,XmmRegister src)471 void AssemblerX8664::movss(Type Ty, XmmRegister dst, XmmRegister src) {
472 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
473 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
474 emitRexRB(RexTypeIrrelevant, src, dst);
475 emitUint8(0x0F);
476 emitUint8(0x11);
477 emitXmmRegisterOperand(src, dst);
478 }
479
movd(Type SrcTy,XmmRegister dst,GPRRegister src)480 void AssemblerX8664::movd(Type SrcTy, XmmRegister dst, GPRRegister src) {
481 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
482 emitUint8(0x66);
483 emitRexRB(SrcTy, dst, src);
484 emitUint8(0x0F);
485 emitUint8(0x6E);
486 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
487 }
488
movd(Type SrcTy,XmmRegister dst,const AsmAddress & src)489 void AssemblerX8664::movd(Type SrcTy, XmmRegister dst, const AsmAddress &src) {
490 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
491 emitUint8(0x66);
492 emitRex(SrcTy, src, dst);
493 emitUint8(0x0F);
494 emitUint8(0x6E);
495 emitOperand(gprEncoding(dst), src);
496 }
497
movd(Type DestTy,GPRRegister dst,XmmRegister src)498 void AssemblerX8664::movd(Type DestTy, GPRRegister dst, XmmRegister src) {
499 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
500 emitUint8(0x66);
501 emitRexRB(DestTy, src, dst);
502 emitUint8(0x0F);
503 emitUint8(0x7E);
504 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
505 }
506
movd(Type DestTy,const AsmAddress & dst,XmmRegister src)507 void AssemblerX8664::movd(Type DestTy, const AsmAddress &dst, XmmRegister src) {
508 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
509 emitUint8(0x66);
510 emitRex(DestTy, dst, src);
511 emitUint8(0x0F);
512 emitUint8(0x7E);
513 emitOperand(gprEncoding(src), dst);
514 }
515
movq(XmmRegister dst,XmmRegister src)516 void AssemblerX8664::movq(XmmRegister dst, XmmRegister src) {
517 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
518 emitUint8(0xF3);
519 emitRexRB(RexTypeIrrelevant, dst, src);
520 emitUint8(0x0F);
521 emitUint8(0x7E);
522 emitXmmRegisterOperand(dst, src);
523 }
524
movq(const AsmAddress & dst,XmmRegister src)525 void AssemblerX8664::movq(const AsmAddress &dst, XmmRegister src) {
526 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
527 emitUint8(0x66);
528 emitRex(RexTypeIrrelevant, dst, src);
529 emitUint8(0x0F);
530 emitUint8(0xD6);
531 emitOperand(gprEncoding(src), dst);
532 }
533
movq(XmmRegister dst,const AsmAddress & src)534 void AssemblerX8664::movq(XmmRegister dst, const AsmAddress &src) {
535 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
536 emitUint8(0xF3);
537 emitRex(RexTypeIrrelevant, src, dst);
538 emitUint8(0x0F);
539 emitUint8(0x7E);
540 emitOperand(gprEncoding(dst), src);
541 }
542
addss(Type Ty,XmmRegister dst,XmmRegister src)543 void AssemblerX8664::addss(Type Ty, XmmRegister dst, XmmRegister src) {
544 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
545 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
546 emitRexRB(RexTypeIrrelevant, dst, src);
547 emitUint8(0x0F);
548 emitUint8(0x58);
549 emitXmmRegisterOperand(dst, src);
550 }
551
addss(Type Ty,XmmRegister dst,const AsmAddress & src)552 void AssemblerX8664::addss(Type Ty, XmmRegister dst, const AsmAddress &src) {
553 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
554 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
555 emitRex(RexTypeIrrelevant, src, dst);
556 emitUint8(0x0F);
557 emitUint8(0x58);
558 emitOperand(gprEncoding(dst), src);
559 }
560
subss(Type Ty,XmmRegister dst,XmmRegister src)561 void AssemblerX8664::subss(Type Ty, XmmRegister dst, XmmRegister src) {
562 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
563 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
564 emitRexRB(RexTypeIrrelevant, dst, src);
565 emitUint8(0x0F);
566 emitUint8(0x5C);
567 emitXmmRegisterOperand(dst, src);
568 }
569
subss(Type Ty,XmmRegister dst,const AsmAddress & src)570 void AssemblerX8664::subss(Type Ty, XmmRegister dst, const AsmAddress &src) {
571 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
572 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
573 emitRex(RexTypeIrrelevant, src, dst);
574 emitUint8(0x0F);
575 emitUint8(0x5C);
576 emitOperand(gprEncoding(dst), src);
577 }
578
mulss(Type Ty,XmmRegister dst,XmmRegister src)579 void AssemblerX8664::mulss(Type Ty, XmmRegister dst, XmmRegister src) {
580 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
581 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
582 emitRexRB(RexTypeIrrelevant, dst, src);
583 emitUint8(0x0F);
584 emitUint8(0x59);
585 emitXmmRegisterOperand(dst, src);
586 }
587
mulss(Type Ty,XmmRegister dst,const AsmAddress & src)588 void AssemblerX8664::mulss(Type Ty, XmmRegister dst, const AsmAddress &src) {
589 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
590 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
591 emitRex(RexTypeIrrelevant, src, dst);
592 emitUint8(0x0F);
593 emitUint8(0x59);
594 emitOperand(gprEncoding(dst), src);
595 }
596
divss(Type Ty,XmmRegister dst,XmmRegister src)597 void AssemblerX8664::divss(Type Ty, XmmRegister dst, XmmRegister src) {
598 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
599 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
600 emitRexRB(RexTypeIrrelevant, dst, src);
601 emitUint8(0x0F);
602 emitUint8(0x5E);
603 emitXmmRegisterOperand(dst, src);
604 }
605
divss(Type Ty,XmmRegister dst,const AsmAddress & src)606 void AssemblerX8664::divss(Type Ty, XmmRegister dst, const AsmAddress &src) {
607 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
608 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
609 emitRex(RexTypeIrrelevant, src, dst);
610 emitUint8(0x0F);
611 emitUint8(0x5E);
612 emitOperand(gprEncoding(dst), src);
613 }
614
movaps(XmmRegister dst,XmmRegister src)615 void AssemblerX8664::movaps(XmmRegister dst, XmmRegister src) {
616 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
617 emitRexRB(RexTypeIrrelevant, dst, src);
618 emitUint8(0x0F);
619 emitUint8(0x28);
620 emitXmmRegisterOperand(dst, src);
621 }
622
movups(XmmRegister dst,XmmRegister src)623 void AssemblerX8664::movups(XmmRegister dst, XmmRegister src) {
624 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
625 emitRexRB(RexTypeIrrelevant, dst, src);
626 emitUint8(0x0F);
627 emitUint8(0x10);
628 emitXmmRegisterOperand(dst, src);
629 }
630
movups(XmmRegister dst,const AsmAddress & src)631 void AssemblerX8664::movups(XmmRegister dst, const AsmAddress &src) {
632 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
633 emitRex(RexTypeIrrelevant, src, dst);
634 emitUint8(0x0F);
635 emitUint8(0x10);
636 emitOperand(gprEncoding(dst), src);
637 }
638
movups(const AsmAddress & dst,XmmRegister src)639 void AssemblerX8664::movups(const AsmAddress &dst, XmmRegister src) {
640 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
641 emitRex(RexTypeIrrelevant, dst, src);
642 emitUint8(0x0F);
643 emitUint8(0x11);
644 emitOperand(gprEncoding(src), dst);
645 }
646
padd(Type Ty,XmmRegister dst,XmmRegister src)647 void AssemblerX8664::padd(Type Ty, XmmRegister dst, XmmRegister src) {
648 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
649 emitUint8(0x66);
650 emitRexRB(RexTypeIrrelevant, dst, src);
651 emitUint8(0x0F);
652 if (isByteSizedArithType(Ty)) {
653 emitUint8(0xFC);
654 } else if (Ty == IceType_i16) {
655 emitUint8(0xFD);
656 } else {
657 emitUint8(0xFE);
658 }
659 emitXmmRegisterOperand(dst, src);
660 }
661
padd(Type Ty,XmmRegister dst,const AsmAddress & src)662 void AssemblerX8664::padd(Type Ty, XmmRegister dst, const AsmAddress &src) {
663 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
664 emitUint8(0x66);
665 emitRex(RexTypeIrrelevant, src, dst);
666 emitUint8(0x0F);
667 if (isByteSizedArithType(Ty)) {
668 emitUint8(0xFC);
669 } else if (Ty == IceType_i16) {
670 emitUint8(0xFD);
671 } else {
672 emitUint8(0xFE);
673 }
674 emitOperand(gprEncoding(dst), src);
675 }
676
padds(Type Ty,XmmRegister dst,XmmRegister src)677 void AssemblerX8664::padds(Type Ty, XmmRegister dst, XmmRegister src) {
678 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
679 emitUint8(0x66);
680 emitRexRB(RexTypeIrrelevant, dst, src);
681 emitUint8(0x0F);
682 if (isByteSizedArithType(Ty)) {
683 emitUint8(0xEC);
684 } else if (Ty == IceType_i16) {
685 emitUint8(0xED);
686 } else {
687 assert(false && "Unexpected padds operand type");
688 }
689 emitXmmRegisterOperand(dst, src);
690 }
691
padds(Type Ty,XmmRegister dst,const AsmAddress & src)692 void AssemblerX8664::padds(Type Ty, XmmRegister dst, const AsmAddress &src) {
693 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
694 emitUint8(0x66);
695 emitRex(RexTypeIrrelevant, src, dst);
696 emitUint8(0x0F);
697 if (isByteSizedArithType(Ty)) {
698 emitUint8(0xEC);
699 } else if (Ty == IceType_i16) {
700 emitUint8(0xED);
701 } else {
702 assert(false && "Unexpected padds operand type");
703 }
704 emitOperand(gprEncoding(dst), src);
705 }
706
paddus(Type Ty,XmmRegister dst,XmmRegister src)707 void AssemblerX8664::paddus(Type Ty, XmmRegister dst, XmmRegister src) {
708 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
709 emitUint8(0x66);
710 emitRexRB(RexTypeIrrelevant, dst, src);
711 emitUint8(0x0F);
712 if (isByteSizedArithType(Ty)) {
713 emitUint8(0xDC);
714 } else if (Ty == IceType_i16) {
715 emitUint8(0xDD);
716 } else {
717 assert(false && "Unexpected paddus operand type");
718 }
719 emitXmmRegisterOperand(dst, src);
720 }
721
paddus(Type Ty,XmmRegister dst,const AsmAddress & src)722 void AssemblerX8664::paddus(Type Ty, XmmRegister dst, const AsmAddress &src) {
723 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
724 emitUint8(0x66);
725 emitRex(RexTypeIrrelevant, src, dst);
726 emitUint8(0x0F);
727 if (isByteSizedArithType(Ty)) {
728 emitUint8(0xDC);
729 } else if (Ty == IceType_i16) {
730 emitUint8(0xDD);
731 } else {
732 assert(false && "Unexpected paddus operand type");
733 }
734 emitOperand(gprEncoding(dst), src);
735 }
736
pand(Type,XmmRegister dst,XmmRegister src)737 void AssemblerX8664::pand(Type /* Ty */, XmmRegister dst, XmmRegister src) {
738 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
739 emitUint8(0x66);
740 emitRexRB(RexTypeIrrelevant, dst, src);
741 emitUint8(0x0F);
742 emitUint8(0xDB);
743 emitXmmRegisterOperand(dst, src);
744 }
745
pand(Type,XmmRegister dst,const AsmAddress & src)746 void AssemblerX8664::pand(Type /* Ty */, XmmRegister dst,
747 const AsmAddress &src) {
748 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
749 emitUint8(0x66);
750 emitRex(RexTypeIrrelevant, src, dst);
751 emitUint8(0x0F);
752 emitUint8(0xDB);
753 emitOperand(gprEncoding(dst), src);
754 }
755
pandn(Type,XmmRegister dst,XmmRegister src)756 void AssemblerX8664::pandn(Type /* Ty */, XmmRegister dst, XmmRegister src) {
757 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
758 emitUint8(0x66);
759 emitRexRB(RexTypeIrrelevant, dst, src);
760 emitUint8(0x0F);
761 emitUint8(0xDF);
762 emitXmmRegisterOperand(dst, src);
763 }
764
pandn(Type,XmmRegister dst,const AsmAddress & src)765 void AssemblerX8664::pandn(Type /* Ty */, XmmRegister dst,
766 const AsmAddress &src) {
767 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
768 emitUint8(0x66);
769 emitRex(RexTypeIrrelevant, src, dst);
770 emitUint8(0x0F);
771 emitUint8(0xDF);
772 emitOperand(gprEncoding(dst), src);
773 }
774
pmull(Type Ty,XmmRegister dst,XmmRegister src)775 void AssemblerX8664::pmull(Type Ty, XmmRegister dst, XmmRegister src) {
776 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
777 emitUint8(0x66);
778 emitRexRB(RexTypeIrrelevant, dst, src);
779 emitUint8(0x0F);
780 if (Ty == IceType_i16) {
781 emitUint8(0xD5);
782 } else {
783 assert(Ty == IceType_i32);
784 emitUint8(0x38);
785 emitUint8(0x40);
786 }
787 emitXmmRegisterOperand(dst, src);
788 }
789
pmull(Type Ty,XmmRegister dst,const AsmAddress & src)790 void AssemblerX8664::pmull(Type Ty, XmmRegister dst, const AsmAddress &src) {
791 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
792 emitUint8(0x66);
793 emitRex(RexTypeIrrelevant, src, dst);
794 emitUint8(0x0F);
795 if (Ty == IceType_i16) {
796 emitUint8(0xD5);
797 } else {
798 assert(Ty == IceType_i32);
799 emitUint8(0x38);
800 emitUint8(0x40);
801 }
802 emitOperand(gprEncoding(dst), src);
803 }
804
pmulhw(Type Ty,XmmRegister dst,XmmRegister src)805 void AssemblerX8664::pmulhw(Type Ty, XmmRegister dst, XmmRegister src) {
806 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
807 emitUint8(0x66);
808 emitRexRB(RexTypeIrrelevant, dst, src);
809 emitUint8(0x0F);
810 assert(Ty == IceType_v8i16);
811 (void)Ty;
812 emitUint8(0xE5);
813 emitXmmRegisterOperand(dst, src);
814 }
815
pmulhw(Type Ty,XmmRegister dst,const AsmAddress & src)816 void AssemblerX8664::pmulhw(Type Ty, XmmRegister dst, const AsmAddress &src) {
817 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
818 emitUint8(0x66);
819 emitRex(RexTypeIrrelevant, src, dst);
820 emitUint8(0x0F);
821 assert(Ty == IceType_v8i16);
822 (void)Ty;
823 emitUint8(0xE5);
824 emitOperand(gprEncoding(dst), src);
825 }
826
pmulhuw(Type Ty,XmmRegister dst,XmmRegister src)827 void AssemblerX8664::pmulhuw(Type Ty, XmmRegister dst, XmmRegister src) {
828 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
829 emitUint8(0x66);
830 emitRexRB(RexTypeIrrelevant, dst, src);
831 emitUint8(0x0F);
832 assert(Ty == IceType_v8i16);
833 (void)Ty;
834 emitUint8(0xE4);
835 emitXmmRegisterOperand(dst, src);
836 }
837
pmulhuw(Type Ty,XmmRegister dst,const AsmAddress & src)838 void AssemblerX8664::pmulhuw(Type Ty, XmmRegister dst, const AsmAddress &src) {
839 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
840 emitUint8(0x66);
841 emitRex(RexTypeIrrelevant, src, dst);
842 emitUint8(0x0F);
843 assert(Ty == IceType_v8i16);
844 (void)Ty;
845 emitUint8(0xE4);
846 emitOperand(gprEncoding(dst), src);
847 }
848
pmaddwd(Type Ty,XmmRegister dst,XmmRegister src)849 void AssemblerX8664::pmaddwd(Type Ty, XmmRegister dst, XmmRegister src) {
850 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
851 emitUint8(0x66);
852 emitRexRB(RexTypeIrrelevant, dst, src);
853 emitUint8(0x0F);
854 assert(Ty == IceType_v8i16);
855 (void)Ty;
856 emitUint8(0xF5);
857 emitXmmRegisterOperand(dst, src);
858 }
859
pmaddwd(Type Ty,XmmRegister dst,const AsmAddress & src)860 void AssemblerX8664::pmaddwd(Type Ty, XmmRegister dst, const AsmAddress &src) {
861 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
862 emitUint8(0x66);
863 emitRex(RexTypeIrrelevant, src, dst);
864 emitUint8(0x0F);
865 assert(Ty == IceType_v8i16);
866 (void)Ty;
867 emitUint8(0xF5);
868 emitOperand(gprEncoding(dst), src);
869 }
870
pmuludq(Type,XmmRegister dst,XmmRegister src)871 void AssemblerX8664::pmuludq(Type /* Ty */, XmmRegister dst, XmmRegister src) {
872 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
873 emitUint8(0x66);
874 emitRexRB(RexTypeIrrelevant, dst, src);
875 emitUint8(0x0F);
876 emitUint8(0xF4);
877 emitXmmRegisterOperand(dst, src);
878 }
879
pmuludq(Type,XmmRegister dst,const AsmAddress & src)880 void AssemblerX8664::pmuludq(Type /* Ty */, XmmRegister dst,
881 const AsmAddress &src) {
882 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
883 emitUint8(0x66);
884 emitRex(RexTypeIrrelevant, src, dst);
885 emitUint8(0x0F);
886 emitUint8(0xF4);
887 emitOperand(gprEncoding(dst), src);
888 }
889
por(Type,XmmRegister dst,XmmRegister src)890 void AssemblerX8664::por(Type /* Ty */, XmmRegister dst, XmmRegister src) {
891 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
892 emitUint8(0x66);
893 emitRexRB(RexTypeIrrelevant, dst, src);
894 emitUint8(0x0F);
895 emitUint8(0xEB);
896 emitXmmRegisterOperand(dst, src);
897 }
898
por(Type,XmmRegister dst,const AsmAddress & src)899 void AssemblerX8664::por(Type /* Ty */, XmmRegister dst,
900 const AsmAddress &src) {
901 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
902 emitUint8(0x66);
903 emitRex(RexTypeIrrelevant, src, dst);
904 emitUint8(0x0F);
905 emitUint8(0xEB);
906 emitOperand(gprEncoding(dst), src);
907 }
908
psub(Type Ty,XmmRegister dst,XmmRegister src)909 void AssemblerX8664::psub(Type Ty, XmmRegister dst, XmmRegister src) {
910 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
911 emitUint8(0x66);
912 emitRexRB(RexTypeIrrelevant, dst, src);
913 emitUint8(0x0F);
914 if (isByteSizedArithType(Ty)) {
915 emitUint8(0xF8);
916 } else if (Ty == IceType_i16) {
917 emitUint8(0xF9);
918 } else {
919 emitUint8(0xFA);
920 }
921 emitXmmRegisterOperand(dst, src);
922 }
923
psub(Type Ty,XmmRegister dst,const AsmAddress & src)924 void AssemblerX8664::psub(Type Ty, XmmRegister dst, const AsmAddress &src) {
925 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
926 emitUint8(0x66);
927 emitRex(RexTypeIrrelevant, src, dst);
928 emitUint8(0x0F);
929 if (isByteSizedArithType(Ty)) {
930 emitUint8(0xF8);
931 } else if (Ty == IceType_i16) {
932 emitUint8(0xF9);
933 } else {
934 emitUint8(0xFA);
935 }
936 emitOperand(gprEncoding(dst), src);
937 }
938
psubs(Type Ty,XmmRegister dst,XmmRegister src)939 void AssemblerX8664::psubs(Type Ty, XmmRegister dst, XmmRegister src) {
940 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
941 emitUint8(0x66);
942 emitRexRB(RexTypeIrrelevant, dst, src);
943 emitUint8(0x0F);
944 if (isByteSizedArithType(Ty)) {
945 emitUint8(0xE8);
946 } else if (Ty == IceType_i16) {
947 emitUint8(0xE9);
948 } else {
949 assert(false && "Unexpected psubs operand type");
950 }
951 emitXmmRegisterOperand(dst, src);
952 }
953
psubs(Type Ty,XmmRegister dst,const AsmAddress & src)954 void AssemblerX8664::psubs(Type Ty, XmmRegister dst, const AsmAddress &src) {
955 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
956 emitUint8(0x66);
957 emitRex(RexTypeIrrelevant, src, dst);
958 emitUint8(0x0F);
959 if (isByteSizedArithType(Ty)) {
960 emitUint8(0xE8);
961 } else if (Ty == IceType_i16) {
962 emitUint8(0xE9);
963 } else {
964 assert(false && "Unexpected psubs operand type");
965 }
966 emitOperand(gprEncoding(dst), src);
967 }
968
psubus(Type Ty,XmmRegister dst,XmmRegister src)969 void AssemblerX8664::psubus(Type Ty, XmmRegister dst, XmmRegister src) {
970 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
971 emitUint8(0x66);
972 emitRexRB(RexTypeIrrelevant, dst, src);
973 emitUint8(0x0F);
974 if (isByteSizedArithType(Ty)) {
975 emitUint8(0xD8);
976 } else if (Ty == IceType_i16) {
977 emitUint8(0xD9);
978 } else {
979 assert(false && "Unexpected psubus operand type");
980 }
981 emitXmmRegisterOperand(dst, src);
982 }
983
psubus(Type Ty,XmmRegister dst,const AsmAddress & src)984 void AssemblerX8664::psubus(Type Ty, XmmRegister dst, const AsmAddress &src) {
985 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
986 emitUint8(0x66);
987 emitRex(RexTypeIrrelevant, src, dst);
988 emitUint8(0x0F);
989 if (isByteSizedArithType(Ty)) {
990 emitUint8(0xD8);
991 } else if (Ty == IceType_i16) {
992 emitUint8(0xD9);
993 } else {
994 assert(false && "Unexpected psubus operand type");
995 }
996 emitOperand(gprEncoding(dst), src);
997 }
998
pxor(Type,XmmRegister dst,XmmRegister src)999 void AssemblerX8664::pxor(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1000 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1001 emitUint8(0x66);
1002 emitRexRB(RexTypeIrrelevant, dst, src);
1003 emitUint8(0x0F);
1004 emitUint8(0xEF);
1005 emitXmmRegisterOperand(dst, src);
1006 }
1007
pxor(Type,XmmRegister dst,const AsmAddress & src)1008 void AssemblerX8664::pxor(Type /* Ty */, XmmRegister dst,
1009 const AsmAddress &src) {
1010 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1011 emitUint8(0x66);
1012 emitRex(RexTypeIrrelevant, src, dst);
1013 emitUint8(0x0F);
1014 emitUint8(0xEF);
1015 emitOperand(gprEncoding(dst), src);
1016 }
1017
psll(Type Ty,XmmRegister dst,XmmRegister src)1018 void AssemblerX8664::psll(Type Ty, XmmRegister dst, XmmRegister src) {
1019 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1020 emitUint8(0x66);
1021 emitRexRB(RexTypeIrrelevant, dst, src);
1022 emitUint8(0x0F);
1023 if (Ty == IceType_i16) {
1024 emitUint8(0xF1);
1025 } else {
1026 assert(Ty == IceType_i32);
1027 emitUint8(0xF2);
1028 }
1029 emitXmmRegisterOperand(dst, src);
1030 }
1031
psll(Type Ty,XmmRegister dst,const AsmAddress & src)1032 void AssemblerX8664::psll(Type Ty, XmmRegister dst, const AsmAddress &src) {
1033 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1034 emitUint8(0x66);
1035 emitRex(RexTypeIrrelevant, src, dst);
1036 emitUint8(0x0F);
1037 if (Ty == IceType_i16) {
1038 emitUint8(0xF1);
1039 } else {
1040 assert(Ty == IceType_i32);
1041 emitUint8(0xF2);
1042 }
1043 emitOperand(gprEncoding(dst), src);
1044 }
1045
psll(Type Ty,XmmRegister dst,const Immediate & imm)1046 void AssemblerX8664::psll(Type Ty, XmmRegister dst, const Immediate &imm) {
1047 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1048 assert(imm.is_int8());
1049 emitUint8(0x66);
1050 emitRexB(RexTypeIrrelevant, dst);
1051 emitUint8(0x0F);
1052 if (Ty == IceType_i16) {
1053 emitUint8(0x71);
1054 } else {
1055 assert(Ty == IceType_i32);
1056 emitUint8(0x72);
1057 }
1058 emitRegisterOperand(6, gprEncoding(dst));
1059 emitUint8(imm.value() & 0xFF);
1060 }
1061
psra(Type Ty,XmmRegister dst,XmmRegister src)1062 void AssemblerX8664::psra(Type Ty, XmmRegister dst, XmmRegister src) {
1063 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1064 emitUint8(0x66);
1065 emitRexRB(RexTypeIrrelevant, dst, src);
1066 emitUint8(0x0F);
1067 if (Ty == IceType_i16) {
1068 emitUint8(0xE1);
1069 } else {
1070 assert(Ty == IceType_i32);
1071 emitUint8(0xE2);
1072 }
1073 emitXmmRegisterOperand(dst, src);
1074 }
1075
psra(Type Ty,XmmRegister dst,const AsmAddress & src)1076 void AssemblerX8664::psra(Type Ty, XmmRegister dst, const AsmAddress &src) {
1077 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1078 emitUint8(0x66);
1079 emitRex(RexTypeIrrelevant, src, dst);
1080 emitUint8(0x0F);
1081 if (Ty == IceType_i16) {
1082 emitUint8(0xE1);
1083 } else {
1084 assert(Ty == IceType_i32);
1085 emitUint8(0xE2);
1086 }
1087 emitOperand(gprEncoding(dst), src);
1088 }
1089
psra(Type Ty,XmmRegister dst,const Immediate & imm)1090 void AssemblerX8664::psra(Type Ty, XmmRegister dst, const Immediate &imm) {
1091 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1092 assert(imm.is_int8());
1093 emitUint8(0x66);
1094 emitRexB(RexTypeIrrelevant, dst);
1095 emitUint8(0x0F);
1096 if (Ty == IceType_i16) {
1097 emitUint8(0x71);
1098 } else {
1099 assert(Ty == IceType_i32);
1100 emitUint8(0x72);
1101 }
1102 emitRegisterOperand(4, gprEncoding(dst));
1103 emitUint8(imm.value() & 0xFF);
1104 }
1105
psrl(Type Ty,XmmRegister dst,XmmRegister src)1106 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, XmmRegister src) {
1107 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1108 emitUint8(0x66);
1109 emitRexRB(RexTypeIrrelevant, dst, src);
1110 emitUint8(0x0F);
1111 if (Ty == IceType_i16) {
1112 emitUint8(0xD1);
1113 } else if (Ty == IceType_f64) {
1114 emitUint8(0xD3);
1115 } else {
1116 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1117 emitUint8(0xD2);
1118 }
1119 emitXmmRegisterOperand(dst, src);
1120 }
1121
psrl(Type Ty,XmmRegister dst,const AsmAddress & src)1122 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, const AsmAddress &src) {
1123 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1124 emitUint8(0x66);
1125 emitRex(RexTypeIrrelevant, src, dst);
1126 emitUint8(0x0F);
1127 if (Ty == IceType_i16) {
1128 emitUint8(0xD1);
1129 } else if (Ty == IceType_f64) {
1130 emitUint8(0xD3);
1131 } else {
1132 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1133 emitUint8(0xD2);
1134 }
1135 emitOperand(gprEncoding(dst), src);
1136 }
1137
psrl(Type Ty,XmmRegister dst,const Immediate & imm)1138 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, const Immediate &imm) {
1139 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1140 assert(imm.is_int8());
1141 emitUint8(0x66);
1142 emitRexB(RexTypeIrrelevant, dst);
1143 emitUint8(0x0F);
1144 if (Ty == IceType_i16) {
1145 emitUint8(0x71);
1146 } else if (Ty == IceType_f64) {
1147 emitUint8(0x73);
1148 } else {
1149 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1150 emitUint8(0x72);
1151 }
1152 emitRegisterOperand(2, gprEncoding(dst));
1153 emitUint8(imm.value() & 0xFF);
1154 }
1155
1156 // {add,sub,mul,div}ps are given a Ty parameter for consistency with
1157 // {add,sub,mul,div}ss. In the future, when the PNaCl ABI allows addpd, etc.,
1158 // we can use the Ty parameter to decide on adding a 0x66 prefix.
1159
addps(Type,XmmRegister dst,XmmRegister src)1160 void AssemblerX8664::addps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1161 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1162 emitRexRB(RexTypeIrrelevant, dst, src);
1163 emitUint8(0x0F);
1164 emitUint8(0x58);
1165 emitXmmRegisterOperand(dst, src);
1166 }
1167
addps(Type,XmmRegister dst,const AsmAddress & src)1168 void AssemblerX8664::addps(Type /* Ty */, XmmRegister dst,
1169 const AsmAddress &src) {
1170 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1171 emitRex(RexTypeIrrelevant, src, dst);
1172 emitUint8(0x0F);
1173 emitUint8(0x58);
1174 emitOperand(gprEncoding(dst), src);
1175 }
1176
subps(Type,XmmRegister dst,XmmRegister src)1177 void AssemblerX8664::subps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1178 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1179 emitRexRB(RexTypeIrrelevant, dst, src);
1180 emitUint8(0x0F);
1181 emitUint8(0x5C);
1182 emitXmmRegisterOperand(dst, src);
1183 }
1184
subps(Type,XmmRegister dst,const AsmAddress & src)1185 void AssemblerX8664::subps(Type /* Ty */, XmmRegister dst,
1186 const AsmAddress &src) {
1187 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1188 emitRex(RexTypeIrrelevant, src, dst);
1189 emitUint8(0x0F);
1190 emitUint8(0x5C);
1191 emitOperand(gprEncoding(dst), src);
1192 }
1193
divps(Type,XmmRegister dst,XmmRegister src)1194 void AssemblerX8664::divps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1195 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1196 emitRexRB(RexTypeIrrelevant, dst, src);
1197 emitUint8(0x0F);
1198 emitUint8(0x5E);
1199 emitXmmRegisterOperand(dst, src);
1200 }
1201
divps(Type,XmmRegister dst,const AsmAddress & src)1202 void AssemblerX8664::divps(Type /* Ty */, XmmRegister dst,
1203 const AsmAddress &src) {
1204 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1205 emitRex(RexTypeIrrelevant, src, dst);
1206 emitUint8(0x0F);
1207 emitUint8(0x5E);
1208 emitOperand(gprEncoding(dst), src);
1209 }
1210
mulps(Type,XmmRegister dst,XmmRegister src)1211 void AssemblerX8664::mulps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1212 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1213 emitRexRB(RexTypeIrrelevant, dst, src);
1214 emitUint8(0x0F);
1215 emitUint8(0x59);
1216 emitXmmRegisterOperand(dst, src);
1217 }
1218
mulps(Type,XmmRegister dst,const AsmAddress & src)1219 void AssemblerX8664::mulps(Type /* Ty */, XmmRegister dst,
1220 const AsmAddress &src) {
1221 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1222 emitRex(RexTypeIrrelevant, src, dst);
1223 emitUint8(0x0F);
1224 emitUint8(0x59);
1225 emitOperand(gprEncoding(dst), src);
1226 }
1227
minps(Type Ty,XmmRegister dst,XmmRegister src)1228 void AssemblerX8664::minps(Type Ty, XmmRegister dst, XmmRegister src) {
1229 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1230 if (!isFloat32Asserting32Or64(Ty))
1231 emitUint8(0x66);
1232 emitRexRB(RexTypeIrrelevant, dst, src);
1233 emitUint8(0x0F);
1234 emitUint8(0x5D);
1235 emitXmmRegisterOperand(dst, src);
1236 }
1237
minps(Type Ty,XmmRegister dst,const AsmAddress & src)1238 void AssemblerX8664::minps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1239 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1240 if (!isFloat32Asserting32Or64(Ty))
1241 emitUint8(0x66);
1242 emitRex(RexTypeIrrelevant, src, dst);
1243 emitUint8(0x0F);
1244 emitUint8(0x5D);
1245 emitOperand(gprEncoding(dst), src);
1246 }
1247
minss(Type Ty,XmmRegister dst,XmmRegister src)1248 void AssemblerX8664::minss(Type Ty, XmmRegister dst, XmmRegister src) {
1249 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1250 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1251 emitRexRB(RexTypeIrrelevant, dst, src);
1252 emitUint8(0x0F);
1253 emitUint8(0x5D);
1254 emitXmmRegisterOperand(dst, src);
1255 }
1256
minss(Type Ty,XmmRegister dst,const AsmAddress & src)1257 void AssemblerX8664::minss(Type Ty, XmmRegister dst, const AsmAddress &src) {
1258 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1259 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1260 emitRex(RexTypeIrrelevant, src, dst);
1261 emitUint8(0x0F);
1262 emitUint8(0x5D);
1263 emitOperand(gprEncoding(dst), src);
1264 }
1265
maxps(Type Ty,XmmRegister dst,XmmRegister src)1266 void AssemblerX8664::maxps(Type Ty, XmmRegister dst, XmmRegister src) {
1267 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1268 if (!isFloat32Asserting32Or64(Ty))
1269 emitUint8(0x66);
1270 emitRexRB(RexTypeIrrelevant, dst, src);
1271 emitUint8(0x0F);
1272 emitUint8(0x5F);
1273 emitXmmRegisterOperand(dst, src);
1274 }
1275
maxps(Type Ty,XmmRegister dst,const AsmAddress & src)1276 void AssemblerX8664::maxps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1277 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1278 if (!isFloat32Asserting32Or64(Ty))
1279 emitUint8(0x66);
1280 emitRex(RexTypeIrrelevant, src, dst);
1281 emitUint8(0x0F);
1282 emitUint8(0x5F);
1283 emitOperand(gprEncoding(dst), src);
1284 }
1285
maxss(Type Ty,XmmRegister dst,XmmRegister src)1286 void AssemblerX8664::maxss(Type Ty, XmmRegister dst, XmmRegister src) {
1287 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1288 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1289 emitRexRB(RexTypeIrrelevant, dst, src);
1290 emitUint8(0x0F);
1291 emitUint8(0x5F);
1292 emitXmmRegisterOperand(dst, src);
1293 }
1294
maxss(Type Ty,XmmRegister dst,const AsmAddress & src)1295 void AssemblerX8664::maxss(Type Ty, XmmRegister dst, const AsmAddress &src) {
1296 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1297 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1298 emitRex(RexTypeIrrelevant, src, dst);
1299 emitUint8(0x0F);
1300 emitUint8(0x5F);
1301 emitOperand(gprEncoding(dst), src);
1302 }
1303
andnps(Type Ty,XmmRegister dst,XmmRegister src)1304 void AssemblerX8664::andnps(Type Ty, XmmRegister dst, XmmRegister src) {
1305 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1306 if (!isFloat32Asserting32Or64(Ty))
1307 emitUint8(0x66);
1308 emitRexRB(RexTypeIrrelevant, dst, src);
1309 emitUint8(0x0F);
1310 emitUint8(0x55);
1311 emitXmmRegisterOperand(dst, src);
1312 }
1313
andnps(Type Ty,XmmRegister dst,const AsmAddress & src)1314 void AssemblerX8664::andnps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1315 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1316 if (!isFloat32Asserting32Or64(Ty))
1317 emitUint8(0x66);
1318 emitRex(RexTypeIrrelevant, src, dst);
1319 emitUint8(0x0F);
1320 emitUint8(0x55);
1321 emitOperand(gprEncoding(dst), src);
1322 }
1323
andps(Type Ty,XmmRegister dst,XmmRegister src)1324 void AssemblerX8664::andps(Type Ty, XmmRegister dst, XmmRegister src) {
1325 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1326 if (!isFloat32Asserting32Or64(Ty))
1327 emitUint8(0x66);
1328 emitRexRB(RexTypeIrrelevant, dst, src);
1329 emitUint8(0x0F);
1330 emitUint8(0x54);
1331 emitXmmRegisterOperand(dst, src);
1332 }
1333
andps(Type Ty,XmmRegister dst,const AsmAddress & src)1334 void AssemblerX8664::andps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1335 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1336 if (!isFloat32Asserting32Or64(Ty))
1337 emitUint8(0x66);
1338 emitRex(RexTypeIrrelevant, src, dst);
1339 emitUint8(0x0F);
1340 emitUint8(0x54);
1341 emitOperand(gprEncoding(dst), src);
1342 }
1343
orps(Type Ty,XmmRegister dst,XmmRegister src)1344 void AssemblerX8664::orps(Type Ty, XmmRegister dst, XmmRegister src) {
1345 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1346 if (!isFloat32Asserting32Or64(Ty))
1347 emitUint8(0x66);
1348 emitRexRB(RexTypeIrrelevant, dst, src);
1349 emitUint8(0x0F);
1350 emitUint8(0x56);
1351 emitXmmRegisterOperand(dst, src);
1352 }
1353
orps(Type Ty,XmmRegister dst,const AsmAddress & src)1354 void AssemblerX8664::orps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1355 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1356 if (!isFloat32Asserting32Or64(Ty))
1357 emitUint8(0x66);
1358 emitRex(RexTypeIrrelevant, src, dst);
1359 emitUint8(0x0F);
1360 emitUint8(0x56);
1361 emitOperand(gprEncoding(dst), src);
1362 }
1363
blendvps(Type,XmmRegister dst,XmmRegister src)1364 void AssemblerX8664::blendvps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1365 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1366 emitUint8(0x66);
1367 emitRexRB(RexTypeIrrelevant, dst, src);
1368 emitUint8(0x0F);
1369 emitUint8(0x38);
1370 emitUint8(0x14);
1371 emitXmmRegisterOperand(dst, src);
1372 }
1373
blendvps(Type,XmmRegister dst,const AsmAddress & src)1374 void AssemblerX8664::blendvps(Type /* Ty */, XmmRegister dst,
1375 const AsmAddress &src) {
1376 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1377 emitUint8(0x66);
1378 emitRex(RexTypeIrrelevant, src, dst);
1379 emitUint8(0x0F);
1380 emitUint8(0x38);
1381 emitUint8(0x14);
1382 emitOperand(gprEncoding(dst), src);
1383 }
1384
pblendvb(Type,XmmRegister dst,XmmRegister src)1385 void AssemblerX8664::pblendvb(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1386 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1387 emitUint8(0x66);
1388 emitRexRB(RexTypeIrrelevant, dst, src);
1389 emitUint8(0x0F);
1390 emitUint8(0x38);
1391 emitUint8(0x10);
1392 emitXmmRegisterOperand(dst, src);
1393 }
1394
pblendvb(Type,XmmRegister dst,const AsmAddress & src)1395 void AssemblerX8664::pblendvb(Type /* Ty */, XmmRegister dst,
1396 const AsmAddress &src) {
1397 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1398 emitUint8(0x66);
1399 emitRex(RexTypeIrrelevant, src, dst);
1400 emitUint8(0x0F);
1401 emitUint8(0x38);
1402 emitUint8(0x10);
1403 emitOperand(gprEncoding(dst), src);
1404 }
1405
cmpps(Type Ty,XmmRegister dst,XmmRegister src,CmppsCond CmpCondition)1406 void AssemblerX8664::cmpps(Type Ty, XmmRegister dst, XmmRegister src,
1407 CmppsCond CmpCondition) {
1408 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1409 if (Ty == IceType_f64)
1410 emitUint8(0x66);
1411 emitRexRB(RexTypeIrrelevant, dst, src);
1412 emitUint8(0x0F);
1413 emitUint8(0xC2);
1414 emitXmmRegisterOperand(dst, src);
1415 emitUint8(CmpCondition);
1416 }
1417
cmpps(Type Ty,XmmRegister dst,const AsmAddress & src,CmppsCond CmpCondition)1418 void AssemblerX8664::cmpps(Type Ty, XmmRegister dst, const AsmAddress &src,
1419 CmppsCond CmpCondition) {
1420 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1421 if (Ty == IceType_f64)
1422 emitUint8(0x66);
1423 emitRex(RexTypeIrrelevant, src, dst);
1424 emitUint8(0x0F);
1425 emitUint8(0xC2);
1426 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1427 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1428 emitUint8(CmpCondition);
1429 }
1430
sqrtps(XmmRegister dst)1431 void AssemblerX8664::sqrtps(XmmRegister dst) {
1432 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1433 emitRexRB(RexTypeIrrelevant, dst, dst);
1434 emitUint8(0x0F);
1435 emitUint8(0x51);
1436 emitXmmRegisterOperand(dst, dst);
1437 }
1438
rsqrtps(XmmRegister dst)1439 void AssemblerX8664::rsqrtps(XmmRegister dst) {
1440 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1441 emitRexRB(RexTypeIrrelevant, dst, dst);
1442 emitUint8(0x0F);
1443 emitUint8(0x52);
1444 emitXmmRegisterOperand(dst, dst);
1445 }
1446
reciprocalps(XmmRegister dst)1447 void AssemblerX8664::reciprocalps(XmmRegister dst) {
1448 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1449 emitRexRB(RexTypeIrrelevant, dst, dst);
1450 emitUint8(0x0F);
1451 emitUint8(0x53);
1452 emitXmmRegisterOperand(dst, dst);
1453 }
1454
movhlps(XmmRegister dst,XmmRegister src)1455 void AssemblerX8664::movhlps(XmmRegister dst, XmmRegister src) {
1456 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1457 emitRexRB(RexTypeIrrelevant, dst, src);
1458 emitUint8(0x0F);
1459 emitUint8(0x12);
1460 emitXmmRegisterOperand(dst, src);
1461 }
1462
movlhps(XmmRegister dst,XmmRegister src)1463 void AssemblerX8664::movlhps(XmmRegister dst, XmmRegister src) {
1464 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1465 emitRexRB(RexTypeIrrelevant, dst, src);
1466 emitUint8(0x0F);
1467 emitUint8(0x16);
1468 emitXmmRegisterOperand(dst, src);
1469 }
1470
unpcklps(XmmRegister dst,XmmRegister src)1471 void AssemblerX8664::unpcklps(XmmRegister dst, XmmRegister src) {
1472 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1473 emitRexRB(RexTypeIrrelevant, dst, src);
1474 emitUint8(0x0F);
1475 emitUint8(0x14);
1476 emitXmmRegisterOperand(dst, src);
1477 }
1478
unpckhps(XmmRegister dst,XmmRegister src)1479 void AssemblerX8664::unpckhps(XmmRegister dst, XmmRegister src) {
1480 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1481 emitRexRB(RexTypeIrrelevant, dst, src);
1482 emitUint8(0x0F);
1483 emitUint8(0x15);
1484 emitXmmRegisterOperand(dst, src);
1485 }
1486
unpcklpd(XmmRegister dst,XmmRegister src)1487 void AssemblerX8664::unpcklpd(XmmRegister dst, XmmRegister src) {
1488 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1489 emitUint8(0x66);
1490 emitRexRB(RexTypeIrrelevant, dst, src);
1491 emitUint8(0x0F);
1492 emitUint8(0x14);
1493 emitXmmRegisterOperand(dst, src);
1494 }
1495
unpckhpd(XmmRegister dst,XmmRegister src)1496 void AssemblerX8664::unpckhpd(XmmRegister dst, XmmRegister src) {
1497 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1498 emitUint8(0x66);
1499 emitRexRB(RexTypeIrrelevant, dst, src);
1500 emitUint8(0x0F);
1501 emitUint8(0x15);
1502 emitXmmRegisterOperand(dst, src);
1503 }
1504
set1ps(XmmRegister dst,GPRRegister tmp1,const Immediate & imm)1505 void AssemblerX8664::set1ps(XmmRegister dst, GPRRegister tmp1,
1506 const Immediate &imm) {
1507 // Load 32-bit immediate value into tmp1.
1508 mov(IceType_i32, tmp1, imm);
1509 // Move value from tmp1 into dst.
1510 movd(IceType_i32, dst, tmp1);
1511 // Broadcast low lane into other three lanes.
1512 shufps(RexTypeIrrelevant, dst, dst, Immediate(0x0));
1513 }
1514
pshufb(Type,XmmRegister dst,XmmRegister src)1515 void AssemblerX8664::pshufb(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1516 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1517 emitUint8(0x66);
1518 emitRexRB(RexTypeIrrelevant, dst, src);
1519 emitUint8(0x0F);
1520 emitUint8(0x38);
1521 emitUint8(0x00);
1522 emitXmmRegisterOperand(dst, src);
1523 }
1524
pshufb(Type,XmmRegister dst,const AsmAddress & src)1525 void AssemblerX8664::pshufb(Type /* Ty */, XmmRegister dst,
1526 const AsmAddress &src) {
1527 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1528 emitUint8(0x66);
1529 emitRex(RexTypeIrrelevant, src, dst);
1530 emitUint8(0x0F);
1531 emitUint8(0x38);
1532 emitUint8(0x00);
1533 emitOperand(gprEncoding(dst), src);
1534 }
1535
pshufd(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1536 void AssemblerX8664::pshufd(Type /* Ty */, XmmRegister dst, XmmRegister src,
1537 const Immediate &imm) {
1538 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1539 emitUint8(0x66);
1540 emitRexRB(RexTypeIrrelevant, dst, src);
1541 emitUint8(0x0F);
1542 emitUint8(0x70);
1543 emitXmmRegisterOperand(dst, src);
1544 assert(imm.is_uint8());
1545 emitUint8(imm.value());
1546 }
1547
pshufd(Type,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1548 void AssemblerX8664::pshufd(Type /* Ty */, XmmRegister dst,
1549 const AsmAddress &src, const Immediate &imm) {
1550 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1551 emitUint8(0x66);
1552 emitRex(RexTypeIrrelevant, src, dst);
1553 emitUint8(0x0F);
1554 emitUint8(0x70);
1555 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1556 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1557 assert(imm.is_uint8());
1558 emitUint8(imm.value());
1559 }
1560
punpckl(Type Ty,XmmRegister Dst,XmmRegister Src)1561 void AssemblerX8664::punpckl(Type Ty, XmmRegister Dst, XmmRegister Src) {
1562 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1563 emitUint8(0x66);
1564 emitRexRB(RexTypeIrrelevant, Dst, Src);
1565 emitUint8(0x0F);
1566 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1567 emitUint8(0x62);
1568 } else if (Ty == IceType_v8i16) {
1569 emitUint8(0x61);
1570 } else if (Ty == IceType_v16i8) {
1571 emitUint8(0x60);
1572 } else {
1573 assert(false && "Unexpected vector unpack operand type");
1574 }
1575 emitXmmRegisterOperand(Dst, Src);
1576 }
1577
punpckl(Type Ty,XmmRegister Dst,const AsmAddress & Src)1578 void AssemblerX8664::punpckl(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1579 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1580 emitUint8(0x66);
1581 emitRex(RexTypeIrrelevant, Src, Dst);
1582 emitUint8(0x0F);
1583 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1584 emitUint8(0x62);
1585 } else if (Ty == IceType_v8i16) {
1586 emitUint8(0x61);
1587 } else if (Ty == IceType_v16i8) {
1588 emitUint8(0x60);
1589 } else {
1590 assert(false && "Unexpected vector unpack operand type");
1591 }
1592 emitOperand(gprEncoding(Dst), Src);
1593 }
1594
punpckh(Type Ty,XmmRegister Dst,XmmRegister Src)1595 void AssemblerX8664::punpckh(Type Ty, XmmRegister Dst, XmmRegister Src) {
1596 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1597 emitUint8(0x66);
1598 emitRexRB(RexTypeIrrelevant, Dst, Src);
1599 emitUint8(0x0F);
1600 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1601 emitUint8(0x6A);
1602 } else if (Ty == IceType_v8i16) {
1603 emitUint8(0x69);
1604 } else if (Ty == IceType_v16i8) {
1605 emitUint8(0x68);
1606 } else {
1607 assert(false && "Unexpected vector unpack operand type");
1608 }
1609 emitXmmRegisterOperand(Dst, Src);
1610 }
1611
punpckh(Type Ty,XmmRegister Dst,const AsmAddress & Src)1612 void AssemblerX8664::punpckh(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1613 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1614 emitUint8(0x66);
1615 emitRex(RexTypeIrrelevant, Src, Dst);
1616 emitUint8(0x0F);
1617 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1618 emitUint8(0x6A);
1619 } else if (Ty == IceType_v8i16) {
1620 emitUint8(0x69);
1621 } else if (Ty == IceType_v16i8) {
1622 emitUint8(0x68);
1623 } else {
1624 assert(false && "Unexpected vector unpack operand type");
1625 }
1626 emitOperand(gprEncoding(Dst), Src);
1627 }
1628
packss(Type Ty,XmmRegister Dst,XmmRegister Src)1629 void AssemblerX8664::packss(Type Ty, XmmRegister Dst, XmmRegister Src) {
1630 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1631 emitUint8(0x66);
1632 emitRexRB(RexTypeIrrelevant, Dst, Src);
1633 emitUint8(0x0F);
1634 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1635 emitUint8(0x6B);
1636 } else if (Ty == IceType_v8i16) {
1637 emitUint8(0x63);
1638 } else {
1639 assert(false && "Unexpected vector pack operand type");
1640 }
1641 emitXmmRegisterOperand(Dst, Src);
1642 }
1643
packss(Type Ty,XmmRegister Dst,const AsmAddress & Src)1644 void AssemblerX8664::packss(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1645 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1646 emitUint8(0x66);
1647 emitRex(RexTypeIrrelevant, Src, Dst);
1648 emitUint8(0x0F);
1649 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1650 emitUint8(0x6B);
1651 } else if (Ty == IceType_v8i16) {
1652 emitUint8(0x63);
1653 } else {
1654 assert(false && "Unexpected vector pack operand type");
1655 }
1656 emitOperand(gprEncoding(Dst), Src);
1657 }
1658
packus(Type Ty,XmmRegister Dst,XmmRegister Src)1659 void AssemblerX8664::packus(Type Ty, XmmRegister Dst, XmmRegister Src) {
1660 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1661 emitUint8(0x66);
1662 emitRexRB(RexTypeIrrelevant, Dst, Src);
1663 emitUint8(0x0F);
1664 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1665 emitUint8(0x38);
1666 emitUint8(0x2B);
1667 } else if (Ty == IceType_v8i16) {
1668 emitUint8(0x67);
1669 } else {
1670 assert(false && "Unexpected vector pack operand type");
1671 }
1672 emitXmmRegisterOperand(Dst, Src);
1673 }
1674
packus(Type Ty,XmmRegister Dst,const AsmAddress & Src)1675 void AssemblerX8664::packus(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1676 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1677 emitUint8(0x66);
1678 emitRex(RexTypeIrrelevant, Src, Dst);
1679 emitUint8(0x0F);
1680 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1681 emitUint8(0x38);
1682 emitUint8(0x2B);
1683 } else if (Ty == IceType_v8i16) {
1684 emitUint8(0x67);
1685 } else {
1686 assert(false && "Unexpected vector pack operand type");
1687 }
1688 emitOperand(gprEncoding(Dst), Src);
1689 }
1690
shufps(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1691 void AssemblerX8664::shufps(Type /* Ty */, XmmRegister dst, XmmRegister src,
1692 const Immediate &imm) {
1693 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1694 emitRexRB(RexTypeIrrelevant, dst, src);
1695 emitUint8(0x0F);
1696 emitUint8(0xC6);
1697 emitXmmRegisterOperand(dst, src);
1698 assert(imm.is_uint8());
1699 emitUint8(imm.value());
1700 }
1701
shufps(Type,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1702 void AssemblerX8664::shufps(Type /* Ty */, XmmRegister dst,
1703 const AsmAddress &src, const Immediate &imm) {
1704 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1705 emitRex(RexTypeIrrelevant, src, dst);
1706 emitUint8(0x0F);
1707 emitUint8(0xC6);
1708 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1709 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1710 assert(imm.is_uint8());
1711 emitUint8(imm.value());
1712 }
1713
sqrtpd(XmmRegister dst)1714 void AssemblerX8664::sqrtpd(XmmRegister dst) {
1715 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1716 emitUint8(0x66);
1717 emitRexRB(RexTypeIrrelevant, dst, dst);
1718 emitUint8(0x0F);
1719 emitUint8(0x51);
1720 emitXmmRegisterOperand(dst, dst);
1721 }
1722
cvtdq2ps(Type,XmmRegister dst,XmmRegister src)1723 void AssemblerX8664::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
1724 XmmRegister src) {
1725 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1726 emitRexRB(RexTypeIrrelevant, dst, src);
1727 emitUint8(0x0F);
1728 emitUint8(0x5B);
1729 emitXmmRegisterOperand(dst, src);
1730 }
1731
cvtdq2ps(Type,XmmRegister dst,const AsmAddress & src)1732 void AssemblerX8664::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
1733 const AsmAddress &src) {
1734 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1735 emitRex(RexTypeIrrelevant, src, dst);
1736 emitUint8(0x0F);
1737 emitUint8(0x5B);
1738 emitOperand(gprEncoding(dst), src);
1739 }
1740
cvttps2dq(Type,XmmRegister dst,XmmRegister src)1741 void AssemblerX8664::cvttps2dq(Type /* Ignore */, XmmRegister dst,
1742 XmmRegister src) {
1743 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1744 emitUint8(0xF3);
1745 emitRexRB(RexTypeIrrelevant, dst, src);
1746 emitUint8(0x0F);
1747 emitUint8(0x5B);
1748 emitXmmRegisterOperand(dst, src);
1749 }
1750
cvttps2dq(Type,XmmRegister dst,const AsmAddress & src)1751 void AssemblerX8664::cvttps2dq(Type /* Ignore */, XmmRegister dst,
1752 const AsmAddress &src) {
1753 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1754 emitUint8(0xF3);
1755 emitRex(RexTypeIrrelevant, src, dst);
1756 emitUint8(0x0F);
1757 emitUint8(0x5B);
1758 emitOperand(gprEncoding(dst), src);
1759 }
1760
cvtps2dq(Type,XmmRegister dst,XmmRegister src)1761 void AssemblerX8664::cvtps2dq(Type /* Ignore */, XmmRegister dst,
1762 XmmRegister src) {
1763 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1764 emitUint8(0x66);
1765 emitRexRB(RexTypeIrrelevant, dst, src);
1766 emitUint8(0x0F);
1767 emitUint8(0x5B);
1768 emitXmmRegisterOperand(dst, src);
1769 }
1770
cvtps2dq(Type,XmmRegister dst,const AsmAddress & src)1771 void AssemblerX8664::cvtps2dq(Type /* Ignore */, XmmRegister dst,
1772 const AsmAddress &src) {
1773 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1774 emitUint8(0x66);
1775 emitRex(RexTypeIrrelevant, src, dst);
1776 emitUint8(0x0F);
1777 emitUint8(0x5B);
1778 emitOperand(gprEncoding(dst), src);
1779 }
1780
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,GPRRegister src)1781 void AssemblerX8664::cvtsi2ss(Type DestTy, XmmRegister dst, Type SrcTy,
1782 GPRRegister src) {
1783 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1784 emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
1785 emitRexRB(SrcTy, dst, src);
1786 emitUint8(0x0F);
1787 emitUint8(0x2A);
1788 emitXmmRegisterOperand(dst, src);
1789 }
1790
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,const AsmAddress & src)1791 void AssemblerX8664::cvtsi2ss(Type DestTy, XmmRegister dst, Type SrcTy,
1792 const AsmAddress &src) {
1793 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1794 emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
1795 emitRex(SrcTy, src, dst);
1796 emitUint8(0x0F);
1797 emitUint8(0x2A);
1798 emitOperand(gprEncoding(dst), src);
1799 }
1800
cvtfloat2float(Type SrcTy,XmmRegister dst,XmmRegister src)1801 void AssemblerX8664::cvtfloat2float(Type SrcTy, XmmRegister dst,
1802 XmmRegister src) {
1803 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1804 // ss2sd or sd2ss
1805 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1806 emitRexRB(RexTypeIrrelevant, dst, src);
1807 emitUint8(0x0F);
1808 emitUint8(0x5A);
1809 emitXmmRegisterOperand(dst, src);
1810 }
1811
cvtfloat2float(Type SrcTy,XmmRegister dst,const AsmAddress & src)1812 void AssemblerX8664::cvtfloat2float(Type SrcTy, XmmRegister dst,
1813 const AsmAddress &src) {
1814 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1815 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1816 emitRex(RexTypeIrrelevant, src, dst);
1817 emitUint8(0x0F);
1818 emitUint8(0x5A);
1819 emitOperand(gprEncoding(dst), src);
1820 }
1821
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)1822 void AssemblerX8664::cvttss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1823 XmmRegister src) {
1824 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1825 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1826 emitRexRB(DestTy, dst, src);
1827 emitUint8(0x0F);
1828 emitUint8(0x2C);
1829 emitXmmRegisterOperand(dst, src);
1830 }
1831
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,const AsmAddress & src)1832 void AssemblerX8664::cvttss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1833 const AsmAddress &src) {
1834 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1835 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1836 emitRex(DestTy, src, dst);
1837 emitUint8(0x0F);
1838 emitUint8(0x2C);
1839 emitOperand(gprEncoding(dst), src);
1840 }
1841
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)1842 void AssemblerX8664::cvtss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1843 XmmRegister src) {
1844 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1845 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1846 emitRexRB(DestTy, dst, src);
1847 emitUint8(0x0F);
1848 emitUint8(0x2D);
1849 emitXmmRegisterOperand(dst, src);
1850 }
1851
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,const AsmAddress & src)1852 void AssemblerX8664::cvtss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1853 const AsmAddress &src) {
1854 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1855 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1856 emitRex(DestTy, src, dst);
1857 emitUint8(0x0F);
1858 emitUint8(0x2D);
1859 emitOperand(gprEncoding(dst), src);
1860 }
1861
ucomiss(Type Ty,XmmRegister a,XmmRegister b)1862 void AssemblerX8664::ucomiss(Type Ty, XmmRegister a, XmmRegister b) {
1863 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1864 if (Ty == IceType_f64)
1865 emitUint8(0x66);
1866 emitRexRB(RexTypeIrrelevant, a, b);
1867 emitUint8(0x0F);
1868 emitUint8(0x2E);
1869 emitXmmRegisterOperand(a, b);
1870 }
1871
ucomiss(Type Ty,XmmRegister a,const AsmAddress & b)1872 void AssemblerX8664::ucomiss(Type Ty, XmmRegister a, const AsmAddress &b) {
1873 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1874 if (Ty == IceType_f64)
1875 emitUint8(0x66);
1876 emitRex(RexTypeIrrelevant, b, a);
1877 emitUint8(0x0F);
1878 emitUint8(0x2E);
1879 emitOperand(gprEncoding(a), b);
1880 }
1881
movmsk(Type Ty,GPRRegister dst,XmmRegister src)1882 void AssemblerX8664::movmsk(Type Ty, GPRRegister dst, XmmRegister src) {
1883 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1884 if (Ty == IceType_v16i8) {
1885 emitUint8(0x66);
1886 } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
1887 // No operand size prefix
1888 } else {
1889 assert(false && "Unexpected movmsk operand type");
1890 }
1891 emitRexRB(RexTypeIrrelevant, dst, src);
1892 emitUint8(0x0F);
1893 if (Ty == IceType_v16i8) {
1894 emitUint8(0xD7);
1895 } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
1896 emitUint8(0x50);
1897 } else {
1898 assert(false && "Unexpected movmsk operand type");
1899 }
1900 emitXmmRegisterOperand(dst, src);
1901 }
1902
sqrt(Type Ty,XmmRegister dst,const AsmAddress & src)1903 void AssemblerX8664::sqrt(Type Ty, XmmRegister dst, const AsmAddress &src) {
1904 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1905 if (isScalarFloatingType(Ty))
1906 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1907 emitRex(RexTypeIrrelevant, src, dst);
1908 emitUint8(0x0F);
1909 emitUint8(0x51);
1910 emitOperand(gprEncoding(dst), src);
1911 }
1912
sqrt(Type Ty,XmmRegister dst,XmmRegister src)1913 void AssemblerX8664::sqrt(Type Ty, XmmRegister dst, XmmRegister src) {
1914 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1915 if (isScalarFloatingType(Ty))
1916 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1917 emitRexRB(RexTypeIrrelevant, dst, src);
1918 emitUint8(0x0F);
1919 emitUint8(0x51);
1920 emitXmmRegisterOperand(dst, src);
1921 }
1922
xorps(Type Ty,XmmRegister dst,const AsmAddress & src)1923 void AssemblerX8664::xorps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1924 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1925 if (!isFloat32Asserting32Or64(Ty))
1926 emitUint8(0x66);
1927 emitRex(RexTypeIrrelevant, src, dst);
1928 emitUint8(0x0F);
1929 emitUint8(0x57);
1930 emitOperand(gprEncoding(dst), src);
1931 }
1932
xorps(Type Ty,XmmRegister dst,XmmRegister src)1933 void AssemblerX8664::xorps(Type Ty, XmmRegister dst, XmmRegister src) {
1934 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1935 if (!isFloat32Asserting32Or64(Ty))
1936 emitUint8(0x66);
1937 emitRexRB(RexTypeIrrelevant, dst, src);
1938 emitUint8(0x0F);
1939 emitUint8(0x57);
1940 emitXmmRegisterOperand(dst, src);
1941 }
1942
insertps(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & imm)1943 void AssemblerX8664::insertps(Type Ty, XmmRegister dst, XmmRegister src,
1944 const Immediate &imm) {
1945 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1946 assert(imm.is_uint8());
1947 assert(isVectorFloatingType(Ty));
1948 (void)Ty;
1949 emitUint8(0x66);
1950 emitRexRB(RexTypeIrrelevant, dst, src);
1951 emitUint8(0x0F);
1952 emitUint8(0x3A);
1953 emitUint8(0x21);
1954 emitXmmRegisterOperand(dst, src);
1955 emitUint8(imm.value());
1956 }
1957
insertps(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1958 void AssemblerX8664::insertps(Type Ty, XmmRegister dst, const AsmAddress &src,
1959 const Immediate &imm) {
1960 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1961 assert(imm.is_uint8());
1962 assert(isVectorFloatingType(Ty));
1963 (void)Ty;
1964 emitUint8(0x66);
1965 emitRex(RexTypeIrrelevant, src, dst);
1966 emitUint8(0x0F);
1967 emitUint8(0x3A);
1968 emitUint8(0x21);
1969 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1970 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1971 emitUint8(imm.value());
1972 }
1973
pinsr(Type Ty,XmmRegister dst,GPRRegister src,const Immediate & imm)1974 void AssemblerX8664::pinsr(Type Ty, XmmRegister dst, GPRRegister src,
1975 const Immediate &imm) {
1976 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1977 assert(imm.is_uint8());
1978 emitUint8(0x66);
1979 emitRexRB(Ty, dst, src);
1980 emitUint8(0x0F);
1981 if (Ty == IceType_i16) {
1982 emitUint8(0xC4);
1983 } else {
1984 emitUint8(0x3A);
1985 emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
1986 }
1987 emitXmmRegisterOperand(dst, src);
1988 emitUint8(imm.value());
1989 }
1990
pinsr(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1991 void AssemblerX8664::pinsr(Type Ty, XmmRegister dst, const AsmAddress &src,
1992 const Immediate &imm) {
1993 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1994 assert(imm.is_uint8());
1995 emitUint8(0x66);
1996 emitRex(RexTypeIrrelevant, src, dst);
1997 emitUint8(0x0F);
1998 if (Ty == IceType_i16) {
1999 emitUint8(0xC4);
2000 } else {
2001 emitUint8(0x3A);
2002 emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
2003 }
2004 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2005 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
2006 emitUint8(imm.value());
2007 }
2008
pextr(Type Ty,GPRRegister dst,XmmRegister src,const Immediate & imm)2009 void AssemblerX8664::pextr(Type Ty, GPRRegister dst, XmmRegister src,
2010 const Immediate &imm) {
2011 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2012 assert(imm.is_uint8());
2013 if (Ty == IceType_i16) {
2014 emitUint8(0x66);
2015 emitRexRB(Ty, dst, src);
2016 emitUint8(0x0F);
2017 emitUint8(0xC5);
2018 emitXmmRegisterOperand(dst, src);
2019 emitUint8(imm.value());
2020 } else {
2021 emitUint8(0x66);
2022 emitRexRB(Ty, src, dst);
2023 emitUint8(0x0F);
2024 emitUint8(0x3A);
2025 emitUint8(isByteSizedType(Ty) ? 0x14 : 0x16);
2026 // SSE 4.1 versions are "MRI" because dst can be mem, while pextrw (SSE2)
2027 // is RMI because dst must be reg.
2028 emitXmmRegisterOperand(src, dst);
2029 emitUint8(imm.value());
2030 }
2031 }
2032
pmovsxdq(XmmRegister dst,XmmRegister src)2033 void AssemblerX8664::pmovsxdq(XmmRegister dst, XmmRegister src) {
2034 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2035 emitUint8(0x66);
2036 emitRexRB(RexTypeIrrelevant, dst, src);
2037 emitUint8(0x0F);
2038 emitUint8(0x38);
2039 emitUint8(0x25);
2040 emitXmmRegisterOperand(dst, src);
2041 }
2042
pcmpeq(Type Ty,XmmRegister dst,XmmRegister src)2043 void AssemblerX8664::pcmpeq(Type Ty, XmmRegister dst, XmmRegister src) {
2044 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2045 emitUint8(0x66);
2046 emitRexRB(RexTypeIrrelevant, dst, src);
2047 emitUint8(0x0F);
2048 if (isByteSizedArithType(Ty)) {
2049 emitUint8(0x74);
2050 } else if (Ty == IceType_i16) {
2051 emitUint8(0x75);
2052 } else {
2053 emitUint8(0x76);
2054 }
2055 emitXmmRegisterOperand(dst, src);
2056 }
2057
pcmpeq(Type Ty,XmmRegister dst,const AsmAddress & src)2058 void AssemblerX8664::pcmpeq(Type Ty, XmmRegister dst, const AsmAddress &src) {
2059 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2060 emitUint8(0x66);
2061 emitRex(RexTypeIrrelevant, src, dst);
2062 emitUint8(0x0F);
2063 if (isByteSizedArithType(Ty)) {
2064 emitUint8(0x74);
2065 } else if (Ty == IceType_i16) {
2066 emitUint8(0x75);
2067 } else {
2068 emitUint8(0x76);
2069 }
2070 emitOperand(gprEncoding(dst), src);
2071 }
2072
pcmpgt(Type Ty,XmmRegister dst,XmmRegister src)2073 void AssemblerX8664::pcmpgt(Type Ty, XmmRegister dst, XmmRegister src) {
2074 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2075 emitUint8(0x66);
2076 emitRexRB(RexTypeIrrelevant, dst, src);
2077 emitUint8(0x0F);
2078 if (isByteSizedArithType(Ty)) {
2079 emitUint8(0x64);
2080 } else if (Ty == IceType_i16) {
2081 emitUint8(0x65);
2082 } else {
2083 emitUint8(0x66);
2084 }
2085 emitXmmRegisterOperand(dst, src);
2086 }
2087
pcmpgt(Type Ty,XmmRegister dst,const AsmAddress & src)2088 void AssemblerX8664::pcmpgt(Type Ty, XmmRegister dst, const AsmAddress &src) {
2089 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2090 emitUint8(0x66);
2091 emitRex(RexTypeIrrelevant, src, dst);
2092 emitUint8(0x0F);
2093 if (isByteSizedArithType(Ty)) {
2094 emitUint8(0x64);
2095 } else if (Ty == IceType_i16) {
2096 emitUint8(0x65);
2097 } else {
2098 emitUint8(0x66);
2099 }
2100 emitOperand(gprEncoding(dst), src);
2101 }
2102
round(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & mode)2103 void AssemblerX8664::round(Type Ty, XmmRegister dst, XmmRegister src,
2104 const Immediate &mode) {
2105 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2106 emitUint8(0x66);
2107 emitRexRB(RexTypeIrrelevant, dst, src);
2108 emitUint8(0x0F);
2109 emitUint8(0x3A);
2110 switch (Ty) {
2111 case IceType_v4f32:
2112 emitUint8(0x08);
2113 break;
2114 case IceType_f32:
2115 emitUint8(0x0A);
2116 break;
2117 case IceType_f64:
2118 emitUint8(0x0B);
2119 break;
2120 default:
2121 assert(false && "Unsupported round operand type");
2122 }
2123 emitXmmRegisterOperand(dst, src);
2124 // Mask precision exeption.
2125 emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2126 }
2127
round(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & mode)2128 void AssemblerX8664::round(Type Ty, XmmRegister dst, const AsmAddress &src,
2129 const Immediate &mode) {
2130 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2131 emitUint8(0x66);
2132 emitRex(RexTypeIrrelevant, src, dst);
2133 emitUint8(0x0F);
2134 emitUint8(0x3A);
2135 switch (Ty) {
2136 case IceType_v4f32:
2137 emitUint8(0x08);
2138 break;
2139 case IceType_f32:
2140 emitUint8(0x0A);
2141 break;
2142 case IceType_f64:
2143 emitUint8(0x0B);
2144 break;
2145 default:
2146 assert(false && "Unsupported round operand type");
2147 }
2148 emitOperand(gprEncoding(dst), src);
2149 // Mask precision exeption.
2150 emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2151 }
2152
2153 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const Immediate & imm)2154 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg, const Immediate &imm) {
2155 static_assert(Tag < 8, "Tag must be between 0..7");
2156 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2157 if (Ty == IceType_i16)
2158 emitOperandSizeOverride();
2159 emitRexB(Ty, reg);
2160 if (isByteSizedType(Ty)) {
2161 emitComplexI8(Tag, AsmOperand(reg), imm);
2162 } else {
2163 emitComplex(Ty, Tag, AsmOperand(reg), imm);
2164 }
2165 }
2166
2167 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg0,GPRRegister reg1)2168 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg0, GPRRegister reg1) {
2169 static_assert(Tag < 8, "Tag must be between 0..7");
2170 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2171 if (Ty == IceType_i16)
2172 emitOperandSizeOverride();
2173 emitRexRB(Ty, reg0, reg1);
2174 if (isByteSizedType(Ty))
2175 emitUint8(Tag * 8 + 2);
2176 else
2177 emitUint8(Tag * 8 + 3);
2178 emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
2179 }
2180
2181 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const AsmAddress & address)2182 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg,
2183 const AsmAddress &address) {
2184 static_assert(Tag < 8, "Tag must be between 0..7");
2185 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2186 if (Ty == IceType_i16)
2187 emitOperandSizeOverride();
2188 emitRex(Ty, address, reg);
2189 if (isByteSizedType(Ty))
2190 emitUint8(Tag * 8 + 2);
2191 else
2192 emitUint8(Tag * 8 + 3);
2193 emitOperand(gprEncoding(reg), address);
2194 }
2195
2196 template <uint32_t Tag>
arith_int(Type Ty,const AsmAddress & address,GPRRegister reg)2197 void AssemblerX8664::arith_int(Type Ty, const AsmAddress &address,
2198 GPRRegister reg) {
2199 static_assert(Tag < 8, "Tag must be between 0..7");
2200 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2201 if (Ty == IceType_i16)
2202 emitOperandSizeOverride();
2203 emitRex(Ty, address, reg);
2204 if (isByteSizedType(Ty))
2205 emitUint8(Tag * 8 + 0);
2206 else
2207 emitUint8(Tag * 8 + 1);
2208 emitOperand(gprEncoding(reg), address);
2209 }
2210
2211 template <uint32_t Tag>
arith_int(Type Ty,const AsmAddress & address,const Immediate & imm)2212 void AssemblerX8664::arith_int(Type Ty, const AsmAddress &address,
2213 const Immediate &imm) {
2214 static_assert(Tag < 8, "Tag must be between 0..7");
2215 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2216 if (Ty == IceType_i16)
2217 emitOperandSizeOverride();
2218 emitRex(Ty, address, RexRegIrrelevant);
2219 if (isByteSizedType(Ty)) {
2220 emitComplexI8(Tag, address, imm);
2221 } else {
2222 emitComplex(Ty, Tag, address, imm);
2223 }
2224 }
2225
cmp(Type Ty,GPRRegister reg,const Immediate & imm)2226 void AssemblerX8664::cmp(Type Ty, GPRRegister reg, const Immediate &imm) {
2227 arith_int<7>(Ty, reg, imm);
2228 }
2229
cmp(Type Ty,GPRRegister reg0,GPRRegister reg1)2230 void AssemblerX8664::cmp(Type Ty, GPRRegister reg0, GPRRegister reg1) {
2231 arith_int<7>(Ty, reg0, reg1);
2232 }
2233
cmp(Type Ty,GPRRegister reg,const AsmAddress & address)2234 void AssemblerX8664::cmp(Type Ty, GPRRegister reg, const AsmAddress &address) {
2235 arith_int<7>(Ty, reg, address);
2236 }
2237
cmp(Type Ty,const AsmAddress & address,GPRRegister reg)2238 void AssemblerX8664::cmp(Type Ty, const AsmAddress &address, GPRRegister reg) {
2239 arith_int<7>(Ty, address, reg);
2240 }
2241
cmp(Type Ty,const AsmAddress & address,const Immediate & imm)2242 void AssemblerX8664::cmp(Type Ty, const AsmAddress &address,
2243 const Immediate &imm) {
2244 arith_int<7>(Ty, address, imm);
2245 }
2246
test(Type Ty,GPRRegister reg1,GPRRegister reg2)2247 void AssemblerX8664::test(Type Ty, GPRRegister reg1, GPRRegister reg2) {
2248 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2249 if (Ty == IceType_i16)
2250 emitOperandSizeOverride();
2251 emitRexRB(Ty, reg1, reg2);
2252 if (isByteSizedType(Ty))
2253 emitUint8(0x84);
2254 else
2255 emitUint8(0x85);
2256 emitRegisterOperand(gprEncoding(reg1), gprEncoding(reg2));
2257 }
2258
test(Type Ty,const AsmAddress & addr,GPRRegister reg)2259 void AssemblerX8664::test(Type Ty, const AsmAddress &addr, GPRRegister reg) {
2260 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2261 if (Ty == IceType_i16)
2262 emitOperandSizeOverride();
2263 emitRex(Ty, addr, reg);
2264 if (isByteSizedType(Ty))
2265 emitUint8(0x84);
2266 else
2267 emitUint8(0x85);
2268 emitOperand(gprEncoding(reg), addr);
2269 }
2270
test(Type Ty,GPRRegister reg,const Immediate & immediate)2271 void AssemblerX8664::test(Type Ty, GPRRegister reg,
2272 const Immediate &immediate) {
2273 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2274 // For registers that have a byte variant (EAX, EBX, ECX, and EDX) we only
2275 // test the byte register to keep the encoding short. This is legal even if
2276 // the register had high bits set since this only sets flags registers based
2277 // on the "AND" of the two operands, and the immediate had zeros at those
2278 // high bits.
2279 constexpr GPRRegister Last8BitGPR = GPRRegister::Encoded_Reg_r15d;
2280 if (immediate.is_uint8() && reg <= Last8BitGPR) {
2281 // Use zero-extended 8-bit immediate.
2282 emitRexB(Ty, reg);
2283 if (reg == RegX8664::Encoded_Reg_rax) {
2284 emitUint8(0xA8);
2285 } else {
2286 emitUint8(0xF6);
2287 emitUint8(0xC0 + gprEncoding(reg));
2288 }
2289 emitUint8(immediate.value() & 0xFF);
2290 } else if (reg == RegX8664::Encoded_Reg_rax) {
2291 // Use short form if the destination is EAX.
2292 if (Ty == IceType_i16)
2293 emitOperandSizeOverride();
2294 emitUint8(0xA9);
2295 emitImmediate(Ty, immediate);
2296 } else {
2297 if (Ty == IceType_i16)
2298 emitOperandSizeOverride();
2299 emitRexB(Ty, reg);
2300 emitUint8(0xF7);
2301 emitRegisterOperand(0, gprEncoding(reg));
2302 emitImmediate(Ty, immediate);
2303 }
2304 }
2305
test(Type Ty,const AsmAddress & addr,const Immediate & immediate)2306 void AssemblerX8664::test(Type Ty, const AsmAddress &addr,
2307 const Immediate &immediate) {
2308 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2309 // If the immediate is short, we only test the byte addr to keep the encoding
2310 // short.
2311 if (immediate.is_uint8()) {
2312 // Use zero-extended 8-bit immediate.
2313 emitRex(Ty, addr, RexRegIrrelevant);
2314 emitUint8(0xF6);
2315 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2316 emitOperand(0, addr, OffsetFromNextInstruction);
2317 emitUint8(immediate.value() & 0xFF);
2318 } else {
2319 if (Ty == IceType_i16)
2320 emitOperandSizeOverride();
2321 emitRex(Ty, addr, RexRegIrrelevant);
2322 emitUint8(0xF7);
2323 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
2324 emitOperand(0, addr, OffsetFromNextInstruction);
2325 emitImmediate(Ty, immediate);
2326 }
2327 }
2328
And(Type Ty,GPRRegister dst,GPRRegister src)2329 void AssemblerX8664::And(Type Ty, GPRRegister dst, GPRRegister src) {
2330 arith_int<4>(Ty, dst, src);
2331 }
2332
And(Type Ty,GPRRegister dst,const AsmAddress & address)2333 void AssemblerX8664::And(Type Ty, GPRRegister dst, const AsmAddress &address) {
2334 arith_int<4>(Ty, dst, address);
2335 }
2336
And(Type Ty,GPRRegister dst,const Immediate & imm)2337 void AssemblerX8664::And(Type Ty, GPRRegister dst, const Immediate &imm) {
2338 arith_int<4>(Ty, dst, imm);
2339 }
2340
And(Type Ty,const AsmAddress & address,GPRRegister reg)2341 void AssemblerX8664::And(Type Ty, const AsmAddress &address, GPRRegister reg) {
2342 arith_int<4>(Ty, address, reg);
2343 }
2344
And(Type Ty,const AsmAddress & address,const Immediate & imm)2345 void AssemblerX8664::And(Type Ty, const AsmAddress &address,
2346 const Immediate &imm) {
2347 arith_int<4>(Ty, address, imm);
2348 }
2349
Or(Type Ty,GPRRegister dst,GPRRegister src)2350 void AssemblerX8664::Or(Type Ty, GPRRegister dst, GPRRegister src) {
2351 arith_int<1>(Ty, dst, src);
2352 }
2353
Or(Type Ty,GPRRegister dst,const AsmAddress & address)2354 void AssemblerX8664::Or(Type Ty, GPRRegister dst, const AsmAddress &address) {
2355 arith_int<1>(Ty, dst, address);
2356 }
2357
Or(Type Ty,GPRRegister dst,const Immediate & imm)2358 void AssemblerX8664::Or(Type Ty, GPRRegister dst, const Immediate &imm) {
2359 arith_int<1>(Ty, dst, imm);
2360 }
2361
Or(Type Ty,const AsmAddress & address,GPRRegister reg)2362 void AssemblerX8664::Or(Type Ty, const AsmAddress &address, GPRRegister reg) {
2363 arith_int<1>(Ty, address, reg);
2364 }
2365
Or(Type Ty,const AsmAddress & address,const Immediate & imm)2366 void AssemblerX8664::Or(Type Ty, const AsmAddress &address,
2367 const Immediate &imm) {
2368 arith_int<1>(Ty, address, imm);
2369 }
2370
Xor(Type Ty,GPRRegister dst,GPRRegister src)2371 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, GPRRegister src) {
2372 arith_int<6>(Ty, dst, src);
2373 }
2374
Xor(Type Ty,GPRRegister dst,const AsmAddress & address)2375 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, const AsmAddress &address) {
2376 arith_int<6>(Ty, dst, address);
2377 }
2378
Xor(Type Ty,GPRRegister dst,const Immediate & imm)2379 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, const Immediate &imm) {
2380 arith_int<6>(Ty, dst, imm);
2381 }
2382
Xor(Type Ty,const AsmAddress & address,GPRRegister reg)2383 void AssemblerX8664::Xor(Type Ty, const AsmAddress &address, GPRRegister reg) {
2384 arith_int<6>(Ty, address, reg);
2385 }
2386
Xor(Type Ty,const AsmAddress & address,const Immediate & imm)2387 void AssemblerX8664::Xor(Type Ty, const AsmAddress &address,
2388 const Immediate &imm) {
2389 arith_int<6>(Ty, address, imm);
2390 }
2391
add(Type Ty,GPRRegister dst,GPRRegister src)2392 void AssemblerX8664::add(Type Ty, GPRRegister dst, GPRRegister src) {
2393 arith_int<0>(Ty, dst, src);
2394 }
2395
add(Type Ty,GPRRegister reg,const AsmAddress & address)2396 void AssemblerX8664::add(Type Ty, GPRRegister reg, const AsmAddress &address) {
2397 arith_int<0>(Ty, reg, address);
2398 }
2399
add(Type Ty,GPRRegister reg,const Immediate & imm)2400 void AssemblerX8664::add(Type Ty, GPRRegister reg, const Immediate &imm) {
2401 arith_int<0>(Ty, reg, imm);
2402 }
2403
add(Type Ty,const AsmAddress & address,GPRRegister reg)2404 void AssemblerX8664::add(Type Ty, const AsmAddress &address, GPRRegister reg) {
2405 arith_int<0>(Ty, address, reg);
2406 }
2407
add(Type Ty,const AsmAddress & address,const Immediate & imm)2408 void AssemblerX8664::add(Type Ty, const AsmAddress &address,
2409 const Immediate &imm) {
2410 arith_int<0>(Ty, address, imm);
2411 }
2412
adc(Type Ty,GPRRegister dst,GPRRegister src)2413 void AssemblerX8664::adc(Type Ty, GPRRegister dst, GPRRegister src) {
2414 arith_int<2>(Ty, dst, src);
2415 }
2416
adc(Type Ty,GPRRegister dst,const AsmAddress & address)2417 void AssemblerX8664::adc(Type Ty, GPRRegister dst, const AsmAddress &address) {
2418 arith_int<2>(Ty, dst, address);
2419 }
2420
adc(Type Ty,GPRRegister reg,const Immediate & imm)2421 void AssemblerX8664::adc(Type Ty, GPRRegister reg, const Immediate &imm) {
2422 arith_int<2>(Ty, reg, imm);
2423 }
2424
adc(Type Ty,const AsmAddress & address,GPRRegister reg)2425 void AssemblerX8664::adc(Type Ty, const AsmAddress &address, GPRRegister reg) {
2426 arith_int<2>(Ty, address, reg);
2427 }
2428
adc(Type Ty,const AsmAddress & address,const Immediate & imm)2429 void AssemblerX8664::adc(Type Ty, const AsmAddress &address,
2430 const Immediate &imm) {
2431 arith_int<2>(Ty, address, imm);
2432 }
2433
sub(Type Ty,GPRRegister dst,GPRRegister src)2434 void AssemblerX8664::sub(Type Ty, GPRRegister dst, GPRRegister src) {
2435 arith_int<5>(Ty, dst, src);
2436 }
2437
sub(Type Ty,GPRRegister reg,const AsmAddress & address)2438 void AssemblerX8664::sub(Type Ty, GPRRegister reg, const AsmAddress &address) {
2439 arith_int<5>(Ty, reg, address);
2440 }
2441
sub(Type Ty,GPRRegister reg,const Immediate & imm)2442 void AssemblerX8664::sub(Type Ty, GPRRegister reg, const Immediate &imm) {
2443 arith_int<5>(Ty, reg, imm);
2444 }
2445
sub(Type Ty,const AsmAddress & address,GPRRegister reg)2446 void AssemblerX8664::sub(Type Ty, const AsmAddress &address, GPRRegister reg) {
2447 arith_int<5>(Ty, address, reg);
2448 }
2449
sub(Type Ty,const AsmAddress & address,const Immediate & imm)2450 void AssemblerX8664::sub(Type Ty, const AsmAddress &address,
2451 const Immediate &imm) {
2452 arith_int<5>(Ty, address, imm);
2453 }
2454
sbb(Type Ty,GPRRegister dst,GPRRegister src)2455 void AssemblerX8664::sbb(Type Ty, GPRRegister dst, GPRRegister src) {
2456 arith_int<3>(Ty, dst, src);
2457 }
2458
sbb(Type Ty,GPRRegister dst,const AsmAddress & address)2459 void AssemblerX8664::sbb(Type Ty, GPRRegister dst, const AsmAddress &address) {
2460 arith_int<3>(Ty, dst, address);
2461 }
2462
sbb(Type Ty,GPRRegister reg,const Immediate & imm)2463 void AssemblerX8664::sbb(Type Ty, GPRRegister reg, const Immediate &imm) {
2464 arith_int<3>(Ty, reg, imm);
2465 }
2466
sbb(Type Ty,const AsmAddress & address,GPRRegister reg)2467 void AssemblerX8664::sbb(Type Ty, const AsmAddress &address, GPRRegister reg) {
2468 arith_int<3>(Ty, address, reg);
2469 }
2470
sbb(Type Ty,const AsmAddress & address,const Immediate & imm)2471 void AssemblerX8664::sbb(Type Ty, const AsmAddress &address,
2472 const Immediate &imm) {
2473 arith_int<3>(Ty, address, imm);
2474 }
2475
cbw()2476 void AssemblerX8664::cbw() {
2477 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2478 emitOperandSizeOverride();
2479 emitUint8(0x98);
2480 }
2481
cwd()2482 void AssemblerX8664::cwd() {
2483 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2484 emitOperandSizeOverride();
2485 emitUint8(0x99);
2486 }
2487
cdq()2488 void AssemblerX8664::cdq() {
2489 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2490 emitUint8(0x99);
2491 }
2492
cqo()2493 void AssemblerX8664::cqo() {
2494 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2495 emitRexB(RexTypeForceRexW, RexRegIrrelevant);
2496 emitUint8(0x99);
2497 }
2498
div(Type Ty,GPRRegister reg)2499 void AssemblerX8664::div(Type Ty, GPRRegister reg) {
2500 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2501 if (Ty == IceType_i16)
2502 emitOperandSizeOverride();
2503 emitRexB(Ty, reg);
2504 if (isByteSizedArithType(Ty))
2505 emitUint8(0xF6);
2506 else
2507 emitUint8(0xF7);
2508 emitRegisterOperand(6, gprEncoding(reg));
2509 }
2510
div(Type Ty,const AsmAddress & addr)2511 void AssemblerX8664::div(Type Ty, const AsmAddress &addr) {
2512 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2513 if (Ty == IceType_i16)
2514 emitOperandSizeOverride();
2515 emitRex(Ty, addr, RexRegIrrelevant);
2516 if (isByteSizedArithType(Ty))
2517 emitUint8(0xF6);
2518 else
2519 emitUint8(0xF7);
2520 emitOperand(6, addr);
2521 }
2522
idiv(Type Ty,GPRRegister reg)2523 void AssemblerX8664::idiv(Type Ty, GPRRegister reg) {
2524 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2525 if (Ty == IceType_i16)
2526 emitOperandSizeOverride();
2527 emitRexB(Ty, reg);
2528 if (isByteSizedArithType(Ty))
2529 emitUint8(0xF6);
2530 else
2531 emitUint8(0xF7);
2532 emitRegisterOperand(7, gprEncoding(reg));
2533 }
2534
idiv(Type Ty,const AsmAddress & addr)2535 void AssemblerX8664::idiv(Type Ty, const AsmAddress &addr) {
2536 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2537 if (Ty == IceType_i16)
2538 emitOperandSizeOverride();
2539 emitRex(Ty, addr, RexRegIrrelevant);
2540 if (isByteSizedArithType(Ty))
2541 emitUint8(0xF6);
2542 else
2543 emitUint8(0xF7);
2544 emitOperand(7, addr);
2545 }
2546
imul(Type Ty,GPRRegister dst,GPRRegister src)2547 void AssemblerX8664::imul(Type Ty, GPRRegister dst, GPRRegister src) {
2548 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2549 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2550 if (Ty == IceType_i16)
2551 emitOperandSizeOverride();
2552 emitRexRB(Ty, dst, src);
2553 emitUint8(0x0F);
2554 emitUint8(0xAF);
2555 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2556 }
2557
imul(Type Ty,GPRRegister reg,const AsmAddress & address)2558 void AssemblerX8664::imul(Type Ty, GPRRegister reg, const AsmAddress &address) {
2559 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2560 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2561 if (Ty == IceType_i16)
2562 emitOperandSizeOverride();
2563 emitRex(Ty, address, reg);
2564 emitUint8(0x0F);
2565 emitUint8(0xAF);
2566 emitOperand(gprEncoding(reg), address);
2567 }
2568
imul(Type Ty,GPRRegister reg,const Immediate & imm)2569 void AssemblerX8664::imul(Type Ty, GPRRegister reg, const Immediate &imm) {
2570 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2571 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2572 if (Ty == IceType_i16)
2573 emitOperandSizeOverride();
2574 emitRexRB(Ty, reg, reg);
2575 if (imm.is_int8()) {
2576 emitUint8(0x6B);
2577 emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
2578 emitUint8(imm.value() & 0xFF);
2579 } else {
2580 emitUint8(0x69);
2581 emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
2582 emitImmediate(Ty, imm);
2583 }
2584 }
2585
imul(Type Ty,GPRRegister reg)2586 void AssemblerX8664::imul(Type Ty, GPRRegister reg) {
2587 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2588 if (Ty == IceType_i16)
2589 emitOperandSizeOverride();
2590 emitRexB(Ty, reg);
2591 if (isByteSizedArithType(Ty))
2592 emitUint8(0xF6);
2593 else
2594 emitUint8(0xF7);
2595 emitRegisterOperand(5, gprEncoding(reg));
2596 }
2597
imul(Type Ty,const AsmAddress & address)2598 void AssemblerX8664::imul(Type Ty, const AsmAddress &address) {
2599 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2600 if (Ty == IceType_i16)
2601 emitOperandSizeOverride();
2602 emitRex(Ty, address, RexRegIrrelevant);
2603 if (isByteSizedArithType(Ty))
2604 emitUint8(0xF6);
2605 else
2606 emitUint8(0xF7);
2607 emitOperand(5, address);
2608 }
2609
imul(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2610 void AssemblerX8664::imul(Type Ty, GPRRegister dst, GPRRegister src,
2611 const Immediate &imm) {
2612 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2613 assert(Ty == IceType_i16 || Ty == IceType_i32);
2614 if (Ty == IceType_i16)
2615 emitOperandSizeOverride();
2616 emitRexRB(Ty, dst, src);
2617 if (imm.is_int8()) {
2618 emitUint8(0x6B);
2619 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2620 emitUint8(imm.value() & 0xFF);
2621 } else {
2622 emitUint8(0x69);
2623 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2624 emitImmediate(Ty, imm);
2625 }
2626 }
2627
imul(Type Ty,GPRRegister dst,const AsmAddress & address,const Immediate & imm)2628 void AssemblerX8664::imul(Type Ty, GPRRegister dst, const AsmAddress &address,
2629 const Immediate &imm) {
2630 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2631 assert(Ty == IceType_i16 || Ty == IceType_i32);
2632 if (Ty == IceType_i16)
2633 emitOperandSizeOverride();
2634 emitRex(Ty, address, dst);
2635 if (imm.is_int8()) {
2636 emitUint8(0x6B);
2637 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2638 emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
2639 emitUint8(imm.value() & 0xFF);
2640 } else {
2641 emitUint8(0x69);
2642 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
2643 emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
2644 emitImmediate(Ty, imm);
2645 }
2646 }
2647
mul(Type Ty,GPRRegister reg)2648 void AssemblerX8664::mul(Type Ty, GPRRegister reg) {
2649 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2650 if (Ty == IceType_i16)
2651 emitOperandSizeOverride();
2652 emitRexB(Ty, reg);
2653 if (isByteSizedArithType(Ty))
2654 emitUint8(0xF6);
2655 else
2656 emitUint8(0xF7);
2657 emitRegisterOperand(4, gprEncoding(reg));
2658 }
2659
mul(Type Ty,const AsmAddress & address)2660 void AssemblerX8664::mul(Type Ty, const AsmAddress &address) {
2661 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2662 if (Ty == IceType_i16)
2663 emitOperandSizeOverride();
2664 emitRex(Ty, address, RexRegIrrelevant);
2665 if (isByteSizedArithType(Ty))
2666 emitUint8(0xF6);
2667 else
2668 emitUint8(0xF7);
2669 emitOperand(4, address);
2670 }
2671
incl(const AsmAddress & address)2672 void AssemblerX8664::incl(const AsmAddress &address) {
2673 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2674 emitRex(IceType_i32, address, RexRegIrrelevant);
2675 emitUint8(0xFF);
2676 emitOperand(0, address);
2677 }
2678
decl(const AsmAddress & address)2679 void AssemblerX8664::decl(const AsmAddress &address) {
2680 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2681 emitRex(IceType_i32, address, RexRegIrrelevant);
2682 emitUint8(0xFF);
2683 emitOperand(1, address);
2684 }
2685
rol(Type Ty,GPRRegister reg,const Immediate & imm)2686 void AssemblerX8664::rol(Type Ty, GPRRegister reg, const Immediate &imm) {
2687 emitGenericShift(0, Ty, reg, imm);
2688 }
2689
rol(Type Ty,GPRRegister operand,GPRRegister shifter)2690 void AssemblerX8664::rol(Type Ty, GPRRegister operand, GPRRegister shifter) {
2691 emitGenericShift(0, Ty, AsmOperand(operand), shifter);
2692 }
2693
rol(Type Ty,const AsmAddress & operand,GPRRegister shifter)2694 void AssemblerX8664::rol(Type Ty, const AsmAddress &operand,
2695 GPRRegister shifter) {
2696 emitGenericShift(0, Ty, operand, shifter);
2697 }
2698
shl(Type Ty,GPRRegister reg,const Immediate & imm)2699 void AssemblerX8664::shl(Type Ty, GPRRegister reg, const Immediate &imm) {
2700 emitGenericShift(4, Ty, reg, imm);
2701 }
2702
shl(Type Ty,GPRRegister operand,GPRRegister shifter)2703 void AssemblerX8664::shl(Type Ty, GPRRegister operand, GPRRegister shifter) {
2704 emitGenericShift(4, Ty, AsmOperand(operand), shifter);
2705 }
2706
shl(Type Ty,const AsmAddress & operand,GPRRegister shifter)2707 void AssemblerX8664::shl(Type Ty, const AsmAddress &operand,
2708 GPRRegister shifter) {
2709 emitGenericShift(4, Ty, operand, shifter);
2710 }
2711
shr(Type Ty,GPRRegister reg,const Immediate & imm)2712 void AssemblerX8664::shr(Type Ty, GPRRegister reg, const Immediate &imm) {
2713 emitGenericShift(5, Ty, reg, imm);
2714 }
2715
shr(Type Ty,GPRRegister operand,GPRRegister shifter)2716 void AssemblerX8664::shr(Type Ty, GPRRegister operand, GPRRegister shifter) {
2717 emitGenericShift(5, Ty, AsmOperand(operand), shifter);
2718 }
2719
shr(Type Ty,const AsmAddress & operand,GPRRegister shifter)2720 void AssemblerX8664::shr(Type Ty, const AsmAddress &operand,
2721 GPRRegister shifter) {
2722 emitGenericShift(5, Ty, operand, shifter);
2723 }
2724
sar(Type Ty,GPRRegister reg,const Immediate & imm)2725 void AssemblerX8664::sar(Type Ty, GPRRegister reg, const Immediate &imm) {
2726 emitGenericShift(7, Ty, reg, imm);
2727 }
2728
sar(Type Ty,GPRRegister operand,GPRRegister shifter)2729 void AssemblerX8664::sar(Type Ty, GPRRegister operand, GPRRegister shifter) {
2730 emitGenericShift(7, Ty, AsmOperand(operand), shifter);
2731 }
2732
sar(Type Ty,const AsmAddress & address,GPRRegister shifter)2733 void AssemblerX8664::sar(Type Ty, const AsmAddress &address,
2734 GPRRegister shifter) {
2735 emitGenericShift(7, Ty, address, shifter);
2736 }
2737
shld(Type Ty,GPRRegister dst,GPRRegister src)2738 void AssemblerX8664::shld(Type Ty, GPRRegister dst, GPRRegister src) {
2739 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2740 assert(Ty == IceType_i16 || Ty == IceType_i32);
2741 if (Ty == IceType_i16)
2742 emitOperandSizeOverride();
2743 emitRexRB(Ty, src, dst);
2744 emitUint8(0x0F);
2745 emitUint8(0xA5);
2746 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2747 }
2748
shld(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2749 void AssemblerX8664::shld(Type Ty, GPRRegister dst, GPRRegister src,
2750 const Immediate &imm) {
2751 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2752 assert(Ty == IceType_i16 || Ty == IceType_i32);
2753 assert(imm.is_int8());
2754 if (Ty == IceType_i16)
2755 emitOperandSizeOverride();
2756 emitRexRB(Ty, src, dst);
2757 emitUint8(0x0F);
2758 emitUint8(0xA4);
2759 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2760 emitUint8(imm.value() & 0xFF);
2761 }
2762
shld(Type Ty,const AsmAddress & operand,GPRRegister src)2763 void AssemblerX8664::shld(Type Ty, const AsmAddress &operand, GPRRegister src) {
2764 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2765 assert(Ty == IceType_i16 || Ty == IceType_i32);
2766 if (Ty == IceType_i16)
2767 emitOperandSizeOverride();
2768 emitRex(Ty, operand, src);
2769 emitUint8(0x0F);
2770 emitUint8(0xA5);
2771 emitOperand(gprEncoding(src), operand);
2772 }
2773
shrd(Type Ty,GPRRegister dst,GPRRegister src)2774 void AssemblerX8664::shrd(Type Ty, GPRRegister dst, GPRRegister src) {
2775 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2776 assert(Ty == IceType_i16 || Ty == IceType_i32);
2777 if (Ty == IceType_i16)
2778 emitOperandSizeOverride();
2779 emitRexRB(Ty, src, dst);
2780 emitUint8(0x0F);
2781 emitUint8(0xAD);
2782 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2783 }
2784
shrd(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2785 void AssemblerX8664::shrd(Type Ty, GPRRegister dst, GPRRegister src,
2786 const Immediate &imm) {
2787 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2788 assert(Ty == IceType_i16 || Ty == IceType_i32);
2789 assert(imm.is_int8());
2790 if (Ty == IceType_i16)
2791 emitOperandSizeOverride();
2792 emitRexRB(Ty, src, dst);
2793 emitUint8(0x0F);
2794 emitUint8(0xAC);
2795 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2796 emitUint8(imm.value() & 0xFF);
2797 }
2798
shrd(Type Ty,const AsmAddress & dst,GPRRegister src)2799 void AssemblerX8664::shrd(Type Ty, const AsmAddress &dst, GPRRegister src) {
2800 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2801 assert(Ty == IceType_i16 || Ty == IceType_i32);
2802 if (Ty == IceType_i16)
2803 emitOperandSizeOverride();
2804 emitRex(Ty, dst, src);
2805 emitUint8(0x0F);
2806 emitUint8(0xAD);
2807 emitOperand(gprEncoding(src), dst);
2808 }
2809
neg(Type Ty,GPRRegister reg)2810 void AssemblerX8664::neg(Type Ty, GPRRegister reg) {
2811 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2812 if (Ty == IceType_i16)
2813 emitOperandSizeOverride();
2814 emitRexB(Ty, reg);
2815 if (isByteSizedArithType(Ty))
2816 emitUint8(0xF6);
2817 else
2818 emitUint8(0xF7);
2819 emitRegisterOperand(3, gprEncoding(reg));
2820 }
2821
neg(Type Ty,const AsmAddress & addr)2822 void AssemblerX8664::neg(Type Ty, const AsmAddress &addr) {
2823 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2824 if (Ty == IceType_i16)
2825 emitOperandSizeOverride();
2826 emitRex(Ty, addr, RexRegIrrelevant);
2827 if (isByteSizedArithType(Ty))
2828 emitUint8(0xF6);
2829 else
2830 emitUint8(0xF7);
2831 emitOperand(3, addr);
2832 }
2833
notl(GPRRegister reg)2834 void AssemblerX8664::notl(GPRRegister reg) {
2835 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2836 emitRexB(IceType_i32, reg);
2837 emitUint8(0xF7);
2838 emitUint8(0xD0 | gprEncoding(reg));
2839 }
2840
bswap(Type Ty,GPRRegister reg)2841 void AssemblerX8664::bswap(Type Ty, GPRRegister reg) {
2842 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2843 assert(Ty == IceType_i32 || Ty == IceType_i64);
2844 emitRexB(Ty, reg);
2845 emitUint8(0x0F);
2846 emitUint8(0xC8 | gprEncoding(reg));
2847 }
2848
bsf(Type Ty,GPRRegister dst,GPRRegister src)2849 void AssemblerX8664::bsf(Type Ty, GPRRegister dst, GPRRegister src) {
2850 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2851 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2852 if (Ty == IceType_i16)
2853 emitOperandSizeOverride();
2854 emitRexRB(Ty, dst, src);
2855 emitUint8(0x0F);
2856 emitUint8(0xBC);
2857 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2858 }
2859
bsf(Type Ty,GPRRegister dst,const AsmAddress & src)2860 void AssemblerX8664::bsf(Type Ty, GPRRegister dst, const AsmAddress &src) {
2861 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2862 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2863 if (Ty == IceType_i16)
2864 emitOperandSizeOverride();
2865 emitRex(Ty, src, dst);
2866 emitUint8(0x0F);
2867 emitUint8(0xBC);
2868 emitOperand(gprEncoding(dst), src);
2869 }
2870
bsr(Type Ty,GPRRegister dst,GPRRegister src)2871 void AssemblerX8664::bsr(Type Ty, GPRRegister dst, GPRRegister src) {
2872 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2873 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2874 if (Ty == IceType_i16)
2875 emitOperandSizeOverride();
2876 emitRexRB(Ty, dst, src);
2877 emitUint8(0x0F);
2878 emitUint8(0xBD);
2879 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2880 }
2881
bsr(Type Ty,GPRRegister dst,const AsmAddress & src)2882 void AssemblerX8664::bsr(Type Ty, GPRRegister dst, const AsmAddress &src) {
2883 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2884 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2885 if (Ty == IceType_i16)
2886 emitOperandSizeOverride();
2887 emitRex(Ty, src, dst);
2888 emitUint8(0x0F);
2889 emitUint8(0xBD);
2890 emitOperand(gprEncoding(dst), src);
2891 }
2892
bt(GPRRegister base,GPRRegister offset)2893 void AssemblerX8664::bt(GPRRegister base, GPRRegister offset) {
2894 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2895 emitRexRB(IceType_i32, offset, base);
2896 emitUint8(0x0F);
2897 emitUint8(0xA3);
2898 emitRegisterOperand(gprEncoding(offset), gprEncoding(base));
2899 }
2900
ret()2901 void AssemblerX8664::ret() {
2902 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2903 emitUint8(0xC3);
2904 }
2905
ret(const Immediate & imm)2906 void AssemblerX8664::ret(const Immediate &imm) {
2907 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2908 emitUint8(0xC2);
2909 assert(imm.is_uint16());
2910 emitUint8(imm.value() & 0xFF);
2911 emitUint8((imm.value() >> 8) & 0xFF);
2912 }
2913
nop(int size)2914 void AssemblerX8664::nop(int size) {
2915 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2916 // There are nops up to size 15, but for now just provide up to size 8.
2917 assert(0 < size && size <= MAX_NOP_SIZE);
2918 switch (size) {
2919 case 1:
2920 emitUint8(0x90);
2921 break;
2922 case 2:
2923 emitUint8(0x66);
2924 emitUint8(0x90);
2925 break;
2926 case 3:
2927 emitUint8(0x0F);
2928 emitUint8(0x1F);
2929 emitUint8(0x00);
2930 break;
2931 case 4:
2932 emitUint8(0x0F);
2933 emitUint8(0x1F);
2934 emitUint8(0x40);
2935 emitUint8(0x00);
2936 break;
2937 case 5:
2938 emitUint8(0x0F);
2939 emitUint8(0x1F);
2940 emitUint8(0x44);
2941 emitUint8(0x00);
2942 emitUint8(0x00);
2943 break;
2944 case 6:
2945 emitUint8(0x66);
2946 emitUint8(0x0F);
2947 emitUint8(0x1F);
2948 emitUint8(0x44);
2949 emitUint8(0x00);
2950 emitUint8(0x00);
2951 break;
2952 case 7:
2953 emitUint8(0x0F);
2954 emitUint8(0x1F);
2955 emitUint8(0x80);
2956 emitUint8(0x00);
2957 emitUint8(0x00);
2958 emitUint8(0x00);
2959 emitUint8(0x00);
2960 break;
2961 case 8:
2962 emitUint8(0x0F);
2963 emitUint8(0x1F);
2964 emitUint8(0x84);
2965 emitUint8(0x00);
2966 emitUint8(0x00);
2967 emitUint8(0x00);
2968 emitUint8(0x00);
2969 emitUint8(0x00);
2970 break;
2971 default:
2972 llvm_unreachable("Unimplemented");
2973 }
2974 }
2975
int3()2976 void AssemblerX8664::int3() {
2977 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2978 emitUint8(0xCC);
2979 }
2980
hlt()2981 void AssemblerX8664::hlt() {
2982 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2983 emitUint8(0xF4);
2984 }
2985
ud2()2986 void AssemblerX8664::ud2() {
2987 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2988 emitUint8(0x0F);
2989 emitUint8(0x0B);
2990 }
2991
j(BrCond condition,Label * label,bool near)2992 void AssemblerX8664::j(BrCond condition, Label *label, bool near) {
2993 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2994 if (label->isBound()) {
2995 static const int kShortSize = 2;
2996 static const int kLongSize = 6;
2997 intptr_t offset = label->getPosition() - Buffer.size();
2998 assert(offset <= 0);
2999 if (Utils::IsInt(8, offset - kShortSize)) {
3000 emitUint8(0x70 + condition);
3001 emitUint8((offset - kShortSize) & 0xFF);
3002 } else {
3003 emitUint8(0x0F);
3004 emitUint8(0x80 + condition);
3005 emitInt32(offset - kLongSize);
3006 }
3007 } else if (near) {
3008 emitUint8(0x70 + condition);
3009 emitNearLabelLink(label);
3010 } else {
3011 emitUint8(0x0F);
3012 emitUint8(0x80 + condition);
3013 emitLabelLink(label);
3014 }
3015 }
3016
j(BrCond condition,const ConstantRelocatable * label)3017 void AssemblerX8664::j(BrCond condition, const ConstantRelocatable *label) {
3018 llvm::report_fatal_error("Untested - please verify and then reenable.");
3019 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3020 emitUint8(0x0F);
3021 emitUint8(0x80 + condition);
3022 auto *Fixup = this->createFixup(FK_PcRel, label);
3023 Fixup->set_addend(-4);
3024 emitFixup(Fixup);
3025 emitInt32(0);
3026 }
3027
jmp(GPRRegister reg)3028 void AssemblerX8664::jmp(GPRRegister reg) {
3029 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3030 emitRexB(RexTypeIrrelevant, reg);
3031 emitUint8(0xFF);
3032 emitRegisterOperand(4, gprEncoding(reg));
3033 }
3034
jmp(Label * label,bool near)3035 void AssemblerX8664::jmp(Label *label, bool near) {
3036 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3037 if (label->isBound()) {
3038 static const int kShortSize = 2;
3039 static const int kLongSize = 5;
3040 intptr_t offset = label->getPosition() - Buffer.size();
3041 assert(offset <= 0);
3042 if (Utils::IsInt(8, offset - kShortSize)) {
3043 emitUint8(0xEB);
3044 emitUint8((offset - kShortSize) & 0xFF);
3045 } else {
3046 emitUint8(0xE9);
3047 emitInt32(offset - kLongSize);
3048 }
3049 } else if (near) {
3050 emitUint8(0xEB);
3051 emitNearLabelLink(label);
3052 } else {
3053 emitUint8(0xE9);
3054 emitLabelLink(label);
3055 }
3056 }
3057
jmp(const ConstantRelocatable * label)3058 void AssemblerX8664::jmp(const ConstantRelocatable *label) {
3059 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3060 emitUint8(0xE9);
3061 auto *Fixup = this->createFixup(FK_PcRel, label);
3062 Fixup->set_addend(-4);
3063 emitFixup(Fixup);
3064 emitInt32(0);
3065 }
3066
jmp(const Immediate & abs_address)3067 void AssemblerX8664::jmp(const Immediate &abs_address) {
3068 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3069 emitUint8(0xE9);
3070 AssemblerFixup *Fixup = createFixup(FK_PcRel, AssemblerFixup::NullSymbol);
3071 Fixup->set_addend(abs_address.value() - 4);
3072 emitFixup(Fixup);
3073 emitInt32(0);
3074 }
3075
mfence()3076 void AssemblerX8664::mfence() {
3077 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3078 emitUint8(0x0F);
3079 emitUint8(0xAE);
3080 emitUint8(0xF0);
3081 }
3082
lock()3083 void AssemblerX8664::lock() {
3084 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3085 emitUint8(0xF0);
3086 }
3087
cmpxchg(Type Ty,const AsmAddress & address,GPRRegister reg,bool Locked)3088 void AssemblerX8664::cmpxchg(Type Ty, const AsmAddress &address,
3089 GPRRegister reg, bool Locked) {
3090 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3091 if (Ty == IceType_i16)
3092 emitOperandSizeOverride();
3093 if (Locked)
3094 emitUint8(0xF0);
3095 emitRex(Ty, address, reg);
3096 emitUint8(0x0F);
3097 if (isByteSizedArithType(Ty))
3098 emitUint8(0xB0);
3099 else
3100 emitUint8(0xB1);
3101 emitOperand(gprEncoding(reg), address);
3102 }
3103
cmpxchg8b(const AsmAddress & address,bool Locked)3104 void AssemblerX8664::cmpxchg8b(const AsmAddress &address, bool Locked) {
3105 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3106 if (Locked)
3107 emitUint8(0xF0);
3108 emitRex(IceType_i32, address, RexRegIrrelevant);
3109 emitUint8(0x0F);
3110 emitUint8(0xC7);
3111 emitOperand(1, address);
3112 }
3113
xadd(Type Ty,const AsmAddress & addr,GPRRegister reg,bool Locked)3114 void AssemblerX8664::xadd(Type Ty, const AsmAddress &addr, GPRRegister reg,
3115 bool Locked) {
3116 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3117 if (Ty == IceType_i16)
3118 emitOperandSizeOverride();
3119 if (Locked)
3120 emitUint8(0xF0);
3121 emitRex(Ty, addr, reg);
3122 emitUint8(0x0F);
3123 if (isByteSizedArithType(Ty))
3124 emitUint8(0xC0);
3125 else
3126 emitUint8(0xC1);
3127 emitOperand(gprEncoding(reg), addr);
3128 }
3129
xchg(Type Ty,GPRRegister reg0,GPRRegister reg1)3130 void AssemblerX8664::xchg(Type Ty, GPRRegister reg0, GPRRegister reg1) {
3131 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3132 if (Ty == IceType_i16)
3133 emitOperandSizeOverride();
3134 // Use short form if either register is RAX.
3135 if (reg0 == RegX8664::Encoded_Reg_rax) {
3136 emitRexB(Ty, reg1);
3137 emitUint8(0x90 + gprEncoding(reg1));
3138 } else if (reg1 == RegX8664::Encoded_Reg_rax) {
3139 emitRexB(Ty, reg0);
3140 emitUint8(0x90 + gprEncoding(reg0));
3141 } else {
3142 emitRexRB(Ty, reg0, reg1);
3143 if (isByteSizedArithType(Ty))
3144 emitUint8(0x86);
3145 else
3146 emitUint8(0x87);
3147 emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
3148 }
3149 }
3150
xchg(Type Ty,const AsmAddress & addr,GPRRegister reg)3151 void AssemblerX8664::xchg(Type Ty, const AsmAddress &addr, GPRRegister reg) {
3152 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3153 if (Ty == IceType_i16)
3154 emitOperandSizeOverride();
3155 emitRex(Ty, addr, reg);
3156 if (isByteSizedArithType(Ty))
3157 emitUint8(0x86);
3158 else
3159 emitUint8(0x87);
3160 emitOperand(gprEncoding(reg), addr);
3161 }
3162
iaca_start()3163 void AssemblerX8664::iaca_start() {
3164 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3165 emitUint8(0x0F);
3166 emitUint8(0x0B);
3167
3168 // mov $111, ebx
3169 constexpr GPRRegister dst = GPRRegister::Encoded_Reg_ebx;
3170 constexpr Type Ty = IceType_i32;
3171 emitRexB(Ty, dst);
3172 emitUint8(0xB8 + gprEncoding(dst));
3173 emitImmediate(Ty, Immediate(111));
3174
3175 emitUint8(0x64);
3176 emitUint8(0x67);
3177 emitUint8(0x90);
3178 }
3179
iaca_end()3180 void AssemblerX8664::iaca_end() {
3181 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3182
3183 // mov $222, ebx
3184 constexpr GPRRegister dst = GPRRegister::Encoded_Reg_ebx;
3185 constexpr Type Ty = IceType_i32;
3186 emitRexB(Ty, dst);
3187 emitUint8(0xB8 + gprEncoding(dst));
3188 emitImmediate(Ty, Immediate(222));
3189
3190 emitUint8(0x64);
3191 emitUint8(0x67);
3192 emitUint8(0x90);
3193
3194 emitUint8(0x0F);
3195 emitUint8(0x0B);
3196 }
3197
emitSegmentOverride(uint8_t prefix)3198 void AssemblerX8664::emitSegmentOverride(uint8_t prefix) {
3199 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3200 emitUint8(prefix);
3201 }
3202
align(intptr_t alignment,intptr_t offset)3203 void AssemblerX8664::align(intptr_t alignment, intptr_t offset) {
3204 assert(llvm::isPowerOf2_32(alignment));
3205 intptr_t pos = offset + Buffer.getPosition();
3206 intptr_t mod = pos & (alignment - 1);
3207 if (mod == 0) {
3208 return;
3209 }
3210 intptr_t bytes_needed = alignment - mod;
3211 while (bytes_needed > MAX_NOP_SIZE) {
3212 nop(MAX_NOP_SIZE);
3213 bytes_needed -= MAX_NOP_SIZE;
3214 }
3215 if (bytes_needed) {
3216 nop(bytes_needed);
3217 }
3218 assert(((offset + Buffer.getPosition()) & (alignment - 1)) == 0);
3219 }
3220
bind(Label * L)3221 void AssemblerX8664::bind(Label *L) {
3222 const intptr_t Bound = Buffer.size();
3223 assert(!L->isBound()); // Labels can only be bound once.
3224 while (L->isLinked()) {
3225 const intptr_t Position = L->getLinkPosition();
3226 const intptr_t Next = Buffer.load<int32_t>(Position);
3227 const intptr_t Offset = Bound - (Position + 4);
3228 Buffer.store<int32_t>(Position, Offset);
3229 L->Position = Next;
3230 }
3231 while (L->hasNear()) {
3232 intptr_t Position = L->getNearPosition();
3233 const intptr_t Offset = Bound - (Position + 1);
3234 assert(Utils::IsInt(8, Offset));
3235 Buffer.store<int8_t>(Position, Offset);
3236 }
3237 L->bindTo(Bound);
3238 }
3239
emitOperand(int rm,const AsmOperand & operand,RelocOffsetT Addend)3240 void AssemblerX8664::emitOperand(int rm, const AsmOperand &operand,
3241 RelocOffsetT Addend) {
3242 assert(rm >= 0 && rm < 8);
3243 const intptr_t length = operand.length_;
3244 assert(length > 0);
3245 intptr_t displacement_start = 1;
3246 // Emit the ModRM byte updated with the given RM value.
3247 assert((operand.encoding_[0] & 0x38) == 0);
3248 emitUint8(operand.encoding_[0] + (rm << 3));
3249 // Whenever the addressing mode is not register indirect, using esp == 0x4
3250 // as the register operation indicates an SIB byte follows.
3251 if (((operand.encoding_[0] & 0xc0) != 0xc0) &&
3252 ((operand.encoding_[0] & 0x07) == 0x04)) {
3253 emitUint8(operand.encoding_[1]);
3254 displacement_start = 2;
3255 }
3256
3257 AssemblerFixup *Fixup = operand.fixup();
3258 if (Fixup == nullptr) {
3259 for (intptr_t i = displacement_start; i < length; i++) {
3260 emitUint8(operand.encoding_[i]);
3261 }
3262 return;
3263 }
3264
3265 // Emit the fixup, and a dummy 4-byte immediate. Note that the Disp32 in
3266 // operand.encoding_[i, i+1, i+2, i+3] is part of the constant relocatable
3267 // used to create the fixup, so there's no need to add it to the addend.
3268 assert(length - displacement_start == 4);
3269 if (fixupIsPCRel(Fixup->kind())) {
3270 Fixup->set_addend(Fixup->get_addend() - Addend);
3271 } else {
3272 Fixup->set_addend(Fixup->get_addend());
3273 }
3274 emitFixup(Fixup);
3275 emitInt32(0);
3276 }
3277
emitImmediate(Type Ty,const Immediate & imm)3278 void AssemblerX8664::emitImmediate(Type Ty, const Immediate &imm) {
3279 auto *const Fixup = imm.fixup();
3280 if (Ty == IceType_i16) {
3281 assert(Fixup == nullptr);
3282 emitInt16(imm.value());
3283 return;
3284 }
3285
3286 if (Fixup == nullptr) {
3287 emitInt32(imm.value());
3288 return;
3289 }
3290
3291 Fixup->set_addend(Fixup->get_addend() + imm.value());
3292 emitFixup(Fixup);
3293 emitInt32(0);
3294 }
3295
emitComplexI8(int rm,const AsmOperand & operand,const Immediate & immediate)3296 void AssemblerX8664::emitComplexI8(int rm, const AsmOperand &operand,
3297 const Immediate &immediate) {
3298 assert(rm >= 0 && rm < 8);
3299 assert(immediate.is_int8());
3300 if (operand.IsRegister(RegX8664::Encoded_Reg_rax)) {
3301 // Use short form if the destination is al.
3302 emitUint8(0x04 + (rm << 3));
3303 emitUint8(immediate.value() & 0xFF);
3304 } else {
3305 // Use sign-extended 8-bit immediate.
3306 emitUint8(0x80);
3307 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3308 emitOperand(rm, operand, OffsetFromNextInstruction);
3309 emitUint8(immediate.value() & 0xFF);
3310 }
3311 }
3312
emitComplex(Type Ty,int rm,const AsmOperand & operand,const Immediate & immediate)3313 void AssemblerX8664::emitComplex(Type Ty, int rm, const AsmOperand &operand,
3314 const Immediate &immediate) {
3315 assert(rm >= 0 && rm < 8);
3316 if (immediate.is_int8()) {
3317 // Use sign-extended 8-bit immediate.
3318 emitUint8(0x83);
3319 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3320 emitOperand(rm, operand, OffsetFromNextInstruction);
3321 emitUint8(immediate.value() & 0xFF);
3322 } else if (operand.IsRegister(RegX8664::Encoded_Reg_rax)) {
3323 // Use short form if the destination is eax.
3324 emitUint8(0x05 + (rm << 3));
3325 emitImmediate(Ty, immediate);
3326 } else {
3327 emitUint8(0x81);
3328 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
3329 emitOperand(rm, operand, OffsetFromNextInstruction);
3330 emitImmediate(Ty, immediate);
3331 }
3332 }
3333
emitLabel(Label * label,intptr_t instruction_size)3334 void AssemblerX8664::emitLabel(Label *label, intptr_t instruction_size) {
3335 if (label->isBound()) {
3336 intptr_t offset = label->getPosition() - Buffer.size();
3337 assert(offset <= 0);
3338 emitInt32(offset - instruction_size);
3339 } else {
3340 emitLabelLink(label);
3341 }
3342 }
3343
emitLabelLink(Label * Label)3344 void AssemblerX8664::emitLabelLink(Label *Label) {
3345 assert(!Label->isBound());
3346 intptr_t Position = Buffer.size();
3347 emitInt32(Label->Position);
3348 Label->linkTo(*this, Position);
3349 }
3350
emitNearLabelLink(Label * Label)3351 void AssemblerX8664::emitNearLabelLink(Label *Label) {
3352 assert(!Label->isBound());
3353 intptr_t Position = Buffer.size();
3354 emitUint8(0);
3355 Label->nearLinkTo(*this, Position);
3356 }
3357
emitGenericShift(int rm,Type Ty,GPRRegister reg,const Immediate & imm)3358 void AssemblerX8664::emitGenericShift(int rm, Type Ty, GPRRegister reg,
3359 const Immediate &imm) {
3360 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3361 // We don't assert that imm fits into 8 bits; instead, it gets masked below.
3362 // Note that we don't mask it further (e.g. to 5 bits) because we want the
3363 // same processor behavior regardless of whether it's an immediate (masked to
3364 // 8 bits) or in register cl (essentially ecx masked to 8 bits).
3365 if (Ty == IceType_i16)
3366 emitOperandSizeOverride();
3367 emitRexB(Ty, reg);
3368 if (imm.value() == 1) {
3369 emitUint8(isByteSizedArithType(Ty) ? 0xD0 : 0xD1);
3370 emitOperand(rm, AsmOperand(reg));
3371 } else {
3372 emitUint8(isByteSizedArithType(Ty) ? 0xC0 : 0xC1);
3373 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3374 emitOperand(rm, AsmOperand(reg), OffsetFromNextInstruction);
3375 emitUint8(imm.value() & 0xFF);
3376 }
3377 }
3378
emitGenericShift(int rm,Type Ty,const AsmOperand & operand,GPRRegister shifter)3379 void AssemblerX8664::emitGenericShift(int rm, Type Ty,
3380 const AsmOperand &operand,
3381 GPRRegister shifter) {
3382 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3383 assert(shifter == RegX8664::Encoded_Reg_rcx);
3384 (void)shifter;
3385 if (Ty == IceType_i16)
3386 emitOperandSizeOverride();
3387 emitRexB(Ty, operand.rm());
3388 emitUint8(isByteSizedArithType(Ty) ? 0xD2 : 0xD3);
3389 emitOperand(rm, operand);
3390 }
3391
3392 } // end of namespace X8664
3393 } // end of namespace Ice
3394