xref: /aosp_15_r20/external/swiftshader/third_party/subzero/src/IceAssemblerX8664.cpp (revision 03ce13f70fcc45d86ee91b7ee4cab1936a95046e)
1 //===- subzero/src/IceAssemblerX8664Impl.h - base x86 assembler -*- C++ -*-=//
2 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
3 // for details. All rights reserved. Use of this source code is governed by a
4 // BSD-style license that can be found in the LICENSE file.
5 //
6 // Modified by the Subzero authors.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //                        The Subzero Code Generator
11 //
12 // This file is distributed under the University of Illinois Open Source
13 // License. See LICENSE.TXT for details.
14 //
15 //===----------------------------------------------------------------------===//
16 //
17 /// \file
18 /// \brief Implements the AssemblerX8664 class.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "IceAssemblerX8664.h"
23 
24 #include "IceCfg.h"
25 #include "IceCfgNode.h"
26 #include "IceOperand.h"
27 #include "IceTargetLoweringX8664.h"
28 
29 namespace Ice {
30 namespace X8664 {
31 
AsmAddress(const Variable * Var,const TargetX8664 * Target)32 AsmAddress::AsmAddress(const Variable *Var, const TargetX8664 *Target) {
33   if (Var->hasReg())
34     llvm::report_fatal_error("Stack Variable has a register assigned");
35   if (Var->mustHaveReg()) {
36     llvm::report_fatal_error("Infinite-weight Variable (" + Var->getName() +
37                              ") has no register assigned - function " +
38                              Target->getFunc()->getFunctionName());
39   }
40   int32_t Offset = Var->getStackOffset();
41   auto BaseRegNum = Var->getBaseRegNum();
42   if (Var->getBaseRegNum().hasNoValue()) {
43     // If the stack pointer needs alignment, we must use the frame pointer for
44     // arguments. For locals, getFrameOrStackReg will return the stack pointer
45     // in this case.
46     if (Target->needsStackPointerAlignment() && Var->getIsArg()) {
47       assert(Target->hasFramePointer());
48       BaseRegNum = Target->getFrameReg();
49     } else {
50       BaseRegNum = Target->getFrameOrStackReg();
51     }
52   }
53   SetBase(RegX8664::getEncodedGPR(BaseRegNum), Offset, AssemblerFixup::NoFixup);
54 }
55 
AsmAddress(const X86OperandMem * Mem,Ice::Assembler * Asm,const Ice::TargetLowering * Target)56 AsmAddress::AsmAddress(const X86OperandMem *Mem, Ice::Assembler *Asm,
57                        const Ice::TargetLowering *Target) {
58   int32_t Disp = 0;
59   if (Mem->getBase() && Mem->getBase()->isRematerializable()) {
60     Disp += Mem->getBase()->getRematerializableOffset(Target);
61   }
62   // The index should never be rematerializable.  But if we ever allow it, then
63   // we should make sure the rematerialization offset is shifted by the Shift
64   // value.
65   assert(!Mem->getIndex() || !Mem->getIndex()->isRematerializable());
66 
67   AssemblerFixup *Fixup = nullptr;
68   // Determine the offset (is it relocatable?)
69   if (Mem->getOffset() != nullptr) {
70     if (const auto *CI = llvm::dyn_cast<ConstantInteger32>(Mem->getOffset())) {
71       Disp += static_cast<int32_t>(CI->getValue());
72     } else if (const auto *CR =
73                    llvm::dyn_cast<ConstantRelocatable>(Mem->getOffset())) {
74       const auto FixupKind =
75           (Mem->getBase() != nullptr || Mem->getIndex() != nullptr) ? FK_Abs
76                                                                     : FK_PcRel;
77       const RelocOffsetT DispAdjustment = FixupKind == FK_PcRel ? 4 : 0;
78       Fixup = Asm->createFixup(FixupKind, CR);
79       Fixup->set_addend(-DispAdjustment);
80       Disp = CR->getOffset();
81     } else {
82       llvm_unreachable("Unexpected offset type");
83     }
84   }
85 
86   // Now convert to the various possible forms.
87   if (Mem->getBase() && Mem->getIndex()) {
88     SetBaseIndex(RegX8664::getEncodedGPR(Mem->getBase()->getRegNum()),
89                  RegX8664::getEncodedGPR(Mem->getIndex()->getRegNum()),
90                  ScaleFactor(Mem->getShift()), Disp, Fixup);
91   } else if (Mem->getBase()) {
92     SetBase(RegX8664::getEncodedGPR(Mem->getBase()->getRegNum()), Disp, Fixup);
93   } else if (Mem->getIndex()) {
94     SetIndex(RegX8664::getEncodedGPR(Mem->getIndex()->getRegNum()),
95              ScaleFactor(Mem->getShift()), Disp, Fixup);
96   } else if (Fixup == nullptr) {
97     SetAbsolute(Disp);
98   } else {
99     SetRipRelative(Disp, Fixup);
100   }
101 }
102 
~AssemblerX8664()103 AssemblerX8664::~AssemblerX8664() {
104   if (BuildDefs::asserts()) {
105     for (const Label *Label : CfgNodeLabels) {
106       Label->finalCheck();
107     }
108     for (const Label *Label : LocalLabels) {
109       Label->finalCheck();
110     }
111   }
112 }
113 
alignFunction()114 void AssemblerX8664::alignFunction() {
115   const SizeT Align = 1 << getBundleAlignLog2Bytes();
116   SizeT BytesNeeded = Utils::OffsetToAlignment(Buffer.getPosition(), Align);
117   constexpr SizeT HltSize = 1;
118   while (BytesNeeded > 0) {
119     hlt();
120     BytesNeeded -= HltSize;
121   }
122 }
123 
getOrCreateLabel(SizeT Number,LabelVector & Labels)124 AssemblerX8664::Label *AssemblerX8664::getOrCreateLabel(SizeT Number,
125                                                         LabelVector &Labels) {
126   Label *L = nullptr;
127   if (Number == Labels.size()) {
128     L = new (this->allocate<Label>()) Label();
129     Labels.push_back(L);
130     return L;
131   }
132   if (Number > Labels.size()) {
133     Utils::reserveAndResize(Labels, Number + 1);
134   }
135   L = Labels[Number];
136   if (!L) {
137     L = new (this->allocate<Label>()) Label();
138     Labels[Number] = L;
139   }
140   return L;
141 }
142 
getCfgNodeLabel(SizeT NodeNumber)143 Ice::Label *AssemblerX8664::getCfgNodeLabel(SizeT NodeNumber) {
144   assert(NodeNumber < CfgNodeLabels.size());
145   return CfgNodeLabels[NodeNumber];
146 }
147 
148 AssemblerX8664::Label *
getOrCreateCfgNodeLabel(SizeT NodeNumber)149 AssemblerX8664::getOrCreateCfgNodeLabel(SizeT NodeNumber) {
150   return getOrCreateLabel(NodeNumber, CfgNodeLabels);
151 }
152 
getOrCreateLocalLabel(SizeT Number)153 AssemblerX8664::Label *AssemblerX8664::getOrCreateLocalLabel(SizeT Number) {
154   return getOrCreateLabel(Number, LocalLabels);
155 }
156 
bindCfgNodeLabel(const CfgNode * Node)157 void AssemblerX8664::bindCfgNodeLabel(const CfgNode *Node) {
158   assert(!getPreliminary());
159   Label *L = getOrCreateCfgNodeLabel(Node->getIndex());
160   this->bind(L);
161 }
162 
bindLocalLabel(SizeT Number)163 void AssemblerX8664::bindLocalLabel(SizeT Number) {
164   Label *L = getOrCreateLocalLabel(Number);
165   if (!getPreliminary())
166     this->bind(L);
167 }
168 
call(GPRRegister reg)169 void AssemblerX8664::call(GPRRegister reg) {
170   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
171   emitRexB(RexTypeIrrelevant, reg);
172   emitUint8(0xFF);
173   emitRegisterOperand(2, gprEncoding(reg));
174 }
175 
call(const AsmAddress & address)176 void AssemblerX8664::call(const AsmAddress &address) {
177   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
178   emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
179   emitUint8(0xFF);
180   emitOperand(2, address);
181 }
182 
call(const ConstantRelocatable * label)183 void AssemblerX8664::call(const ConstantRelocatable *label) {
184   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
185   intptr_t call_start = Buffer.getPosition();
186   emitUint8(0xE8);
187   auto *Fixup = this->createFixup(FK_PcRel, label);
188   Fixup->set_addend(-4);
189   emitFixup(Fixup);
190   emitInt32(0);
191   assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
192   (void)call_start;
193 }
194 
call(const Immediate & abs_address)195 void AssemblerX8664::call(const Immediate &abs_address) {
196   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
197   intptr_t call_start = Buffer.getPosition();
198   emitUint8(0xE8);
199   auto *Fixup = this->createFixup(FK_PcRel, AssemblerFixup::NullSymbol);
200   Fixup->set_addend(abs_address.value() - 4);
201   emitFixup(Fixup);
202   emitInt32(0);
203   assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
204   (void)call_start;
205 }
206 
pushl(GPRRegister reg)207 void AssemblerX8664::pushl(GPRRegister reg) {
208   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
209   emitRexB(RexTypeIrrelevant, reg);
210   emitUint8(0x50 + gprEncoding(reg));
211 }
212 
pushl(const Immediate & Imm)213 void AssemblerX8664::pushl(const Immediate &Imm) {
214   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
215   emitUint8(0x68);
216   emitInt32(Imm.value());
217 }
218 
pushl(const ConstantRelocatable * Label)219 void AssemblerX8664::pushl(const ConstantRelocatable *Label) {
220   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
221   emitUint8(0x68);
222   emitFixup(this->createFixup(FK_Abs, Label));
223   // In x86-32, the emitted value is an addend to the relocation. Therefore, we
224   // must emit a 0 (because we're pushing an absolute relocation.)
225   // In x86-64, the emitted value does not matter (the addend lives in the
226   // relocation record as an extra field.)
227   emitInt32(0);
228 }
229 
popl(GPRRegister reg)230 void AssemblerX8664::popl(GPRRegister reg) {
231   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
232   // Any type that would not force a REX prefix to be emitted can be provided
233   // here.
234   emitRexB(RexTypeIrrelevant, reg);
235   emitUint8(0x58 + gprEncoding(reg));
236 }
237 
popl(const AsmAddress & address)238 void AssemblerX8664::popl(const AsmAddress &address) {
239   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
240   emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
241   emitUint8(0x8F);
242   emitOperand(0, address);
243 }
244 
setcc(BrCond condition,ByteRegister dst)245 void AssemblerX8664::setcc(BrCond condition, ByteRegister dst) {
246   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
247   emitRexB(IceType_i8, dst);
248   emitUint8(0x0F);
249   emitUint8(0x90 + condition);
250   emitUint8(0xC0 + gprEncoding(dst));
251 }
252 
setcc(BrCond condition,const AsmAddress & address)253 void AssemblerX8664::setcc(BrCond condition, const AsmAddress &address) {
254   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
255   emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
256   emitUint8(0x0F);
257   emitUint8(0x90 + condition);
258   emitOperand(0, address);
259 }
260 
mov(Type Ty,GPRRegister dst,const Immediate & imm)261 void AssemblerX8664::mov(Type Ty, GPRRegister dst, const Immediate &imm) {
262   assert(Ty != IceType_i64 && "i64 not supported yet.");
263   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
264   if (Ty == IceType_i16)
265     emitOperandSizeOverride();
266   emitRexB(Ty, dst);
267   if (isByteSizedType(Ty)) {
268     emitUint8(0xB0 + gprEncoding(dst));
269     emitUint8(imm.value() & 0xFF);
270   } else {
271     // TODO(jpp): When removing the assertion above ensure that in x86-64 we
272     // emit a 64-bit immediate.
273     emitUint8(0xB8 + gprEncoding(dst));
274     emitImmediate(Ty, imm);
275   }
276 }
277 
mov(Type Ty,GPRRegister dst,GPRRegister src)278 void AssemblerX8664::mov(Type Ty, GPRRegister dst, GPRRegister src) {
279   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
280   if (Ty == IceType_i16)
281     emitOperandSizeOverride();
282   emitRexRB(Ty, src, dst);
283   if (isByteSizedType(Ty)) {
284     emitUint8(0x88);
285   } else {
286     emitUint8(0x89);
287   }
288   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
289 }
290 
mov(Type Ty,GPRRegister dst,const AsmAddress & src)291 void AssemblerX8664::mov(Type Ty, GPRRegister dst, const AsmAddress &src) {
292   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
293   if (Ty == IceType_i16)
294     emitOperandSizeOverride();
295   emitRex(Ty, src, dst);
296   if (isByteSizedType(Ty)) {
297     emitUint8(0x8A);
298   } else {
299     emitUint8(0x8B);
300   }
301   emitOperand(gprEncoding(dst), src);
302 }
303 
mov(Type Ty,const AsmAddress & dst,GPRRegister src)304 void AssemblerX8664::mov(Type Ty, const AsmAddress &dst, GPRRegister src) {
305   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
306   if (Ty == IceType_i16)
307     emitOperandSizeOverride();
308   emitRex(Ty, dst, src);
309   if (isByteSizedType(Ty)) {
310     emitUint8(0x88);
311   } else {
312     emitUint8(0x89);
313   }
314   emitOperand(gprEncoding(src), dst);
315 }
316 
mov(Type Ty,const AsmAddress & dst,const Immediate & imm)317 void AssemblerX8664::mov(Type Ty, const AsmAddress &dst, const Immediate &imm) {
318   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
319   if (Ty == IceType_i16)
320     emitOperandSizeOverride();
321   emitRex(Ty, dst, RexRegIrrelevant);
322   if (isByteSizedType(Ty)) {
323     emitUint8(0xC6);
324     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
325     emitOperand(0, dst, OffsetFromNextInstruction);
326     emitUint8(imm.value() & 0xFF);
327   } else {
328     emitUint8(0xC7);
329     const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
330     emitOperand(0, dst, OffsetFromNextInstruction);
331     emitImmediate(Ty, imm);
332   }
333 }
334 
movabs(const GPRRegister Dst,uint64_t Imm64)335 void AssemblerX8664::movabs(const GPRRegister Dst, uint64_t Imm64) {
336   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
337   const bool NeedsRexW = (Imm64 & ~0xFFFFFFFFull) != 0;
338   const Type RexType = NeedsRexW ? RexTypeForceRexW : RexTypeIrrelevant;
339   emitRexB(RexType, Dst);
340   emitUint8(0xB8 | gprEncoding(Dst));
341   // When emitting Imm64, we don't have to mask out the upper 32 bits for
342   // emitInt32 will/should only emit a 32-bit constant. In reality, we are
343   // paranoid, so we go ahead an mask the upper bits out anyway.
344   emitInt32(Imm64 & 0xFFFFFFFF);
345   if (NeedsRexW)
346     emitInt32((Imm64 >> 32) & 0xFFFFFFFF);
347 }
348 
movzx(Type SrcTy,GPRRegister dst,GPRRegister src)349 void AssemblerX8664::movzx(Type SrcTy, GPRRegister dst, GPRRegister src) {
350   if (SrcTy == IceType_i32) {
351     // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
352     // operand to 64-bit.
353     mov(IceType_i32, dst, src);
354     return;
355   }
356 
357   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
358   bool ByteSized = isByteSizedType(SrcTy);
359   assert(ByteSized || SrcTy == IceType_i16);
360   emitRexRB(RexTypeIrrelevant, dst, SrcTy, src);
361   emitUint8(0x0F);
362   emitUint8(ByteSized ? 0xB6 : 0xB7);
363   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
364 }
365 
movzx(Type SrcTy,GPRRegister dst,const AsmAddress & src)366 void AssemblerX8664::movzx(Type SrcTy, GPRRegister dst, const AsmAddress &src) {
367   if (SrcTy == IceType_i32) {
368     // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
369     // operand to 64-bit.
370     mov(IceType_i32, dst, src);
371     return;
372   }
373 
374   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
375   bool ByteSized = isByteSizedType(SrcTy);
376   assert(ByteSized || SrcTy == IceType_i16);
377   emitRex(SrcTy, src, RexTypeIrrelevant, dst);
378   emitUint8(0x0F);
379   emitUint8(ByteSized ? 0xB6 : 0xB7);
380   emitOperand(gprEncoding(dst), src);
381 }
382 
movsx(Type SrcTy,GPRRegister dst,GPRRegister src)383 void AssemblerX8664::movsx(Type SrcTy, GPRRegister dst, GPRRegister src) {
384   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
385   bool ByteSized = isByteSizedType(SrcTy);
386   emitRexRB(RexTypeForceRexW, dst, SrcTy, src);
387   if (ByteSized || SrcTy == IceType_i16) {
388     emitUint8(0x0F);
389     emitUint8(ByteSized ? 0xBE : 0xBF);
390   } else {
391     assert(SrcTy == IceType_i32);
392     emitUint8(0x63);
393   }
394   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
395 }
396 
movsx(Type SrcTy,GPRRegister dst,const AsmAddress & src)397 void AssemblerX8664::movsx(Type SrcTy, GPRRegister dst, const AsmAddress &src) {
398   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
399   bool ByteSized = isByteSizedType(SrcTy);
400   emitRex(SrcTy, src, RexTypeForceRexW, dst);
401   if (ByteSized || SrcTy == IceType_i16) {
402     emitUint8(0x0F);
403     emitUint8(ByteSized ? 0xBE : 0xBF);
404   } else {
405     assert(SrcTy == IceType_i32);
406     emitUint8(0x63);
407   }
408   emitOperand(gprEncoding(dst), src);
409 }
410 
lea(Type Ty,GPRRegister dst,const AsmAddress & src)411 void AssemblerX8664::lea(Type Ty, GPRRegister dst, const AsmAddress &src) {
412   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
413   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
414   if (Ty == IceType_i16)
415     emitOperandSizeOverride();
416   emitRex(Ty, src, dst);
417   emitUint8(0x8D);
418   emitOperand(gprEncoding(dst), src);
419 }
420 
cmov(Type Ty,BrCond cond,GPRRegister dst,GPRRegister src)421 void AssemblerX8664::cmov(Type Ty, BrCond cond, GPRRegister dst,
422                           GPRRegister src) {
423   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
424   if (Ty == IceType_i16)
425     emitOperandSizeOverride();
426   else
427     assert(Ty == IceType_i32 || Ty == IceType_i64);
428   emitRexRB(Ty, dst, src);
429   emitUint8(0x0F);
430   emitUint8(0x40 + cond);
431   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
432 }
433 
cmov(Type Ty,BrCond cond,GPRRegister dst,const AsmAddress & src)434 void AssemblerX8664::cmov(Type Ty, BrCond cond, GPRRegister dst,
435                           const AsmAddress &src) {
436   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
437   if (Ty == IceType_i16)
438     emitOperandSizeOverride();
439   else
440     assert(Ty == IceType_i32 || Ty == IceType_i64);
441   emitRex(Ty, src, dst);
442   emitUint8(0x0F);
443   emitUint8(0x40 + cond);
444   emitOperand(gprEncoding(dst), src);
445 }
446 
rep_movsb()447 void AssemblerX8664::rep_movsb() {
448   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
449   emitUint8(0xF3);
450   emitUint8(0xA4);
451 }
452 
movss(Type Ty,XmmRegister dst,const AsmAddress & src)453 void AssemblerX8664::movss(Type Ty, XmmRegister dst, const AsmAddress &src) {
454   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
455   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
456   emitRex(RexTypeIrrelevant, src, dst);
457   emitUint8(0x0F);
458   emitUint8(0x10);
459   emitOperand(gprEncoding(dst), src);
460 }
461 
movss(Type Ty,const AsmAddress & dst,XmmRegister src)462 void AssemblerX8664::movss(Type Ty, const AsmAddress &dst, XmmRegister src) {
463   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
464   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
465   emitRex(RexTypeIrrelevant, dst, src);
466   emitUint8(0x0F);
467   emitUint8(0x11);
468   emitOperand(gprEncoding(src), dst);
469 }
470 
movss(Type Ty,XmmRegister dst,XmmRegister src)471 void AssemblerX8664::movss(Type Ty, XmmRegister dst, XmmRegister src) {
472   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
473   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
474   emitRexRB(RexTypeIrrelevant, src, dst);
475   emitUint8(0x0F);
476   emitUint8(0x11);
477   emitXmmRegisterOperand(src, dst);
478 }
479 
movd(Type SrcTy,XmmRegister dst,GPRRegister src)480 void AssemblerX8664::movd(Type SrcTy, XmmRegister dst, GPRRegister src) {
481   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
482   emitUint8(0x66);
483   emitRexRB(SrcTy, dst, src);
484   emitUint8(0x0F);
485   emitUint8(0x6E);
486   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
487 }
488 
movd(Type SrcTy,XmmRegister dst,const AsmAddress & src)489 void AssemblerX8664::movd(Type SrcTy, XmmRegister dst, const AsmAddress &src) {
490   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
491   emitUint8(0x66);
492   emitRex(SrcTy, src, dst);
493   emitUint8(0x0F);
494   emitUint8(0x6E);
495   emitOperand(gprEncoding(dst), src);
496 }
497 
movd(Type DestTy,GPRRegister dst,XmmRegister src)498 void AssemblerX8664::movd(Type DestTy, GPRRegister dst, XmmRegister src) {
499   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
500   emitUint8(0x66);
501   emitRexRB(DestTy, src, dst);
502   emitUint8(0x0F);
503   emitUint8(0x7E);
504   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
505 }
506 
movd(Type DestTy,const AsmAddress & dst,XmmRegister src)507 void AssemblerX8664::movd(Type DestTy, const AsmAddress &dst, XmmRegister src) {
508   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
509   emitUint8(0x66);
510   emitRex(DestTy, dst, src);
511   emitUint8(0x0F);
512   emitUint8(0x7E);
513   emitOperand(gprEncoding(src), dst);
514 }
515 
movq(XmmRegister dst,XmmRegister src)516 void AssemblerX8664::movq(XmmRegister dst, XmmRegister src) {
517   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
518   emitUint8(0xF3);
519   emitRexRB(RexTypeIrrelevant, dst, src);
520   emitUint8(0x0F);
521   emitUint8(0x7E);
522   emitXmmRegisterOperand(dst, src);
523 }
524 
movq(const AsmAddress & dst,XmmRegister src)525 void AssemblerX8664::movq(const AsmAddress &dst, XmmRegister src) {
526   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
527   emitUint8(0x66);
528   emitRex(RexTypeIrrelevant, dst, src);
529   emitUint8(0x0F);
530   emitUint8(0xD6);
531   emitOperand(gprEncoding(src), dst);
532 }
533 
movq(XmmRegister dst,const AsmAddress & src)534 void AssemblerX8664::movq(XmmRegister dst, const AsmAddress &src) {
535   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
536   emitUint8(0xF3);
537   emitRex(RexTypeIrrelevant, src, dst);
538   emitUint8(0x0F);
539   emitUint8(0x7E);
540   emitOperand(gprEncoding(dst), src);
541 }
542 
addss(Type Ty,XmmRegister dst,XmmRegister src)543 void AssemblerX8664::addss(Type Ty, XmmRegister dst, XmmRegister src) {
544   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
545   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
546   emitRexRB(RexTypeIrrelevant, dst, src);
547   emitUint8(0x0F);
548   emitUint8(0x58);
549   emitXmmRegisterOperand(dst, src);
550 }
551 
addss(Type Ty,XmmRegister dst,const AsmAddress & src)552 void AssemblerX8664::addss(Type Ty, XmmRegister dst, const AsmAddress &src) {
553   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
554   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
555   emitRex(RexTypeIrrelevant, src, dst);
556   emitUint8(0x0F);
557   emitUint8(0x58);
558   emitOperand(gprEncoding(dst), src);
559 }
560 
subss(Type Ty,XmmRegister dst,XmmRegister src)561 void AssemblerX8664::subss(Type Ty, XmmRegister dst, XmmRegister src) {
562   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
563   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
564   emitRexRB(RexTypeIrrelevant, dst, src);
565   emitUint8(0x0F);
566   emitUint8(0x5C);
567   emitXmmRegisterOperand(dst, src);
568 }
569 
subss(Type Ty,XmmRegister dst,const AsmAddress & src)570 void AssemblerX8664::subss(Type Ty, XmmRegister dst, const AsmAddress &src) {
571   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
572   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
573   emitRex(RexTypeIrrelevant, src, dst);
574   emitUint8(0x0F);
575   emitUint8(0x5C);
576   emitOperand(gprEncoding(dst), src);
577 }
578 
mulss(Type Ty,XmmRegister dst,XmmRegister src)579 void AssemblerX8664::mulss(Type Ty, XmmRegister dst, XmmRegister src) {
580   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
581   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
582   emitRexRB(RexTypeIrrelevant, dst, src);
583   emitUint8(0x0F);
584   emitUint8(0x59);
585   emitXmmRegisterOperand(dst, src);
586 }
587 
mulss(Type Ty,XmmRegister dst,const AsmAddress & src)588 void AssemblerX8664::mulss(Type Ty, XmmRegister dst, const AsmAddress &src) {
589   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
590   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
591   emitRex(RexTypeIrrelevant, src, dst);
592   emitUint8(0x0F);
593   emitUint8(0x59);
594   emitOperand(gprEncoding(dst), src);
595 }
596 
divss(Type Ty,XmmRegister dst,XmmRegister src)597 void AssemblerX8664::divss(Type Ty, XmmRegister dst, XmmRegister src) {
598   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
599   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
600   emitRexRB(RexTypeIrrelevant, dst, src);
601   emitUint8(0x0F);
602   emitUint8(0x5E);
603   emitXmmRegisterOperand(dst, src);
604 }
605 
divss(Type Ty,XmmRegister dst,const AsmAddress & src)606 void AssemblerX8664::divss(Type Ty, XmmRegister dst, const AsmAddress &src) {
607   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
608   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
609   emitRex(RexTypeIrrelevant, src, dst);
610   emitUint8(0x0F);
611   emitUint8(0x5E);
612   emitOperand(gprEncoding(dst), src);
613 }
614 
movaps(XmmRegister dst,XmmRegister src)615 void AssemblerX8664::movaps(XmmRegister dst, XmmRegister src) {
616   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
617   emitRexRB(RexTypeIrrelevant, dst, src);
618   emitUint8(0x0F);
619   emitUint8(0x28);
620   emitXmmRegisterOperand(dst, src);
621 }
622 
movups(XmmRegister dst,XmmRegister src)623 void AssemblerX8664::movups(XmmRegister dst, XmmRegister src) {
624   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
625   emitRexRB(RexTypeIrrelevant, dst, src);
626   emitUint8(0x0F);
627   emitUint8(0x10);
628   emitXmmRegisterOperand(dst, src);
629 }
630 
movups(XmmRegister dst,const AsmAddress & src)631 void AssemblerX8664::movups(XmmRegister dst, const AsmAddress &src) {
632   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
633   emitRex(RexTypeIrrelevant, src, dst);
634   emitUint8(0x0F);
635   emitUint8(0x10);
636   emitOperand(gprEncoding(dst), src);
637 }
638 
movups(const AsmAddress & dst,XmmRegister src)639 void AssemblerX8664::movups(const AsmAddress &dst, XmmRegister src) {
640   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
641   emitRex(RexTypeIrrelevant, dst, src);
642   emitUint8(0x0F);
643   emitUint8(0x11);
644   emitOperand(gprEncoding(src), dst);
645 }
646 
padd(Type Ty,XmmRegister dst,XmmRegister src)647 void AssemblerX8664::padd(Type Ty, XmmRegister dst, XmmRegister src) {
648   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
649   emitUint8(0x66);
650   emitRexRB(RexTypeIrrelevant, dst, src);
651   emitUint8(0x0F);
652   if (isByteSizedArithType(Ty)) {
653     emitUint8(0xFC);
654   } else if (Ty == IceType_i16) {
655     emitUint8(0xFD);
656   } else {
657     emitUint8(0xFE);
658   }
659   emitXmmRegisterOperand(dst, src);
660 }
661 
padd(Type Ty,XmmRegister dst,const AsmAddress & src)662 void AssemblerX8664::padd(Type Ty, XmmRegister dst, const AsmAddress &src) {
663   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
664   emitUint8(0x66);
665   emitRex(RexTypeIrrelevant, src, dst);
666   emitUint8(0x0F);
667   if (isByteSizedArithType(Ty)) {
668     emitUint8(0xFC);
669   } else if (Ty == IceType_i16) {
670     emitUint8(0xFD);
671   } else {
672     emitUint8(0xFE);
673   }
674   emitOperand(gprEncoding(dst), src);
675 }
676 
padds(Type Ty,XmmRegister dst,XmmRegister src)677 void AssemblerX8664::padds(Type Ty, XmmRegister dst, XmmRegister src) {
678   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
679   emitUint8(0x66);
680   emitRexRB(RexTypeIrrelevant, dst, src);
681   emitUint8(0x0F);
682   if (isByteSizedArithType(Ty)) {
683     emitUint8(0xEC);
684   } else if (Ty == IceType_i16) {
685     emitUint8(0xED);
686   } else {
687     assert(false && "Unexpected padds operand type");
688   }
689   emitXmmRegisterOperand(dst, src);
690 }
691 
padds(Type Ty,XmmRegister dst,const AsmAddress & src)692 void AssemblerX8664::padds(Type Ty, XmmRegister dst, const AsmAddress &src) {
693   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
694   emitUint8(0x66);
695   emitRex(RexTypeIrrelevant, src, dst);
696   emitUint8(0x0F);
697   if (isByteSizedArithType(Ty)) {
698     emitUint8(0xEC);
699   } else if (Ty == IceType_i16) {
700     emitUint8(0xED);
701   } else {
702     assert(false && "Unexpected padds operand type");
703   }
704   emitOperand(gprEncoding(dst), src);
705 }
706 
paddus(Type Ty,XmmRegister dst,XmmRegister src)707 void AssemblerX8664::paddus(Type Ty, XmmRegister dst, XmmRegister src) {
708   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
709   emitUint8(0x66);
710   emitRexRB(RexTypeIrrelevant, dst, src);
711   emitUint8(0x0F);
712   if (isByteSizedArithType(Ty)) {
713     emitUint8(0xDC);
714   } else if (Ty == IceType_i16) {
715     emitUint8(0xDD);
716   } else {
717     assert(false && "Unexpected paddus operand type");
718   }
719   emitXmmRegisterOperand(dst, src);
720 }
721 
paddus(Type Ty,XmmRegister dst,const AsmAddress & src)722 void AssemblerX8664::paddus(Type Ty, XmmRegister dst, const AsmAddress &src) {
723   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
724   emitUint8(0x66);
725   emitRex(RexTypeIrrelevant, src, dst);
726   emitUint8(0x0F);
727   if (isByteSizedArithType(Ty)) {
728     emitUint8(0xDC);
729   } else if (Ty == IceType_i16) {
730     emitUint8(0xDD);
731   } else {
732     assert(false && "Unexpected paddus operand type");
733   }
734   emitOperand(gprEncoding(dst), src);
735 }
736 
pand(Type,XmmRegister dst,XmmRegister src)737 void AssemblerX8664::pand(Type /* Ty */, XmmRegister dst, XmmRegister src) {
738   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
739   emitUint8(0x66);
740   emitRexRB(RexTypeIrrelevant, dst, src);
741   emitUint8(0x0F);
742   emitUint8(0xDB);
743   emitXmmRegisterOperand(dst, src);
744 }
745 
pand(Type,XmmRegister dst,const AsmAddress & src)746 void AssemblerX8664::pand(Type /* Ty */, XmmRegister dst,
747                           const AsmAddress &src) {
748   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
749   emitUint8(0x66);
750   emitRex(RexTypeIrrelevant, src, dst);
751   emitUint8(0x0F);
752   emitUint8(0xDB);
753   emitOperand(gprEncoding(dst), src);
754 }
755 
pandn(Type,XmmRegister dst,XmmRegister src)756 void AssemblerX8664::pandn(Type /* Ty */, XmmRegister dst, XmmRegister src) {
757   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
758   emitUint8(0x66);
759   emitRexRB(RexTypeIrrelevant, dst, src);
760   emitUint8(0x0F);
761   emitUint8(0xDF);
762   emitXmmRegisterOperand(dst, src);
763 }
764 
pandn(Type,XmmRegister dst,const AsmAddress & src)765 void AssemblerX8664::pandn(Type /* Ty */, XmmRegister dst,
766                            const AsmAddress &src) {
767   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
768   emitUint8(0x66);
769   emitRex(RexTypeIrrelevant, src, dst);
770   emitUint8(0x0F);
771   emitUint8(0xDF);
772   emitOperand(gprEncoding(dst), src);
773 }
774 
pmull(Type Ty,XmmRegister dst,XmmRegister src)775 void AssemblerX8664::pmull(Type Ty, XmmRegister dst, XmmRegister src) {
776   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
777   emitUint8(0x66);
778   emitRexRB(RexTypeIrrelevant, dst, src);
779   emitUint8(0x0F);
780   if (Ty == IceType_i16) {
781     emitUint8(0xD5);
782   } else {
783     assert(Ty == IceType_i32);
784     emitUint8(0x38);
785     emitUint8(0x40);
786   }
787   emitXmmRegisterOperand(dst, src);
788 }
789 
pmull(Type Ty,XmmRegister dst,const AsmAddress & src)790 void AssemblerX8664::pmull(Type Ty, XmmRegister dst, const AsmAddress &src) {
791   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
792   emitUint8(0x66);
793   emitRex(RexTypeIrrelevant, src, dst);
794   emitUint8(0x0F);
795   if (Ty == IceType_i16) {
796     emitUint8(0xD5);
797   } else {
798     assert(Ty == IceType_i32);
799     emitUint8(0x38);
800     emitUint8(0x40);
801   }
802   emitOperand(gprEncoding(dst), src);
803 }
804 
pmulhw(Type Ty,XmmRegister dst,XmmRegister src)805 void AssemblerX8664::pmulhw(Type Ty, XmmRegister dst, XmmRegister src) {
806   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
807   emitUint8(0x66);
808   emitRexRB(RexTypeIrrelevant, dst, src);
809   emitUint8(0x0F);
810   assert(Ty == IceType_v8i16);
811   (void)Ty;
812   emitUint8(0xE5);
813   emitXmmRegisterOperand(dst, src);
814 }
815 
pmulhw(Type Ty,XmmRegister dst,const AsmAddress & src)816 void AssemblerX8664::pmulhw(Type Ty, XmmRegister dst, const AsmAddress &src) {
817   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
818   emitUint8(0x66);
819   emitRex(RexTypeIrrelevant, src, dst);
820   emitUint8(0x0F);
821   assert(Ty == IceType_v8i16);
822   (void)Ty;
823   emitUint8(0xE5);
824   emitOperand(gprEncoding(dst), src);
825 }
826 
pmulhuw(Type Ty,XmmRegister dst,XmmRegister src)827 void AssemblerX8664::pmulhuw(Type Ty, XmmRegister dst, XmmRegister src) {
828   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
829   emitUint8(0x66);
830   emitRexRB(RexTypeIrrelevant, dst, src);
831   emitUint8(0x0F);
832   assert(Ty == IceType_v8i16);
833   (void)Ty;
834   emitUint8(0xE4);
835   emitXmmRegisterOperand(dst, src);
836 }
837 
pmulhuw(Type Ty,XmmRegister dst,const AsmAddress & src)838 void AssemblerX8664::pmulhuw(Type Ty, XmmRegister dst, const AsmAddress &src) {
839   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
840   emitUint8(0x66);
841   emitRex(RexTypeIrrelevant, src, dst);
842   emitUint8(0x0F);
843   assert(Ty == IceType_v8i16);
844   (void)Ty;
845   emitUint8(0xE4);
846   emitOperand(gprEncoding(dst), src);
847 }
848 
pmaddwd(Type Ty,XmmRegister dst,XmmRegister src)849 void AssemblerX8664::pmaddwd(Type Ty, XmmRegister dst, XmmRegister src) {
850   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
851   emitUint8(0x66);
852   emitRexRB(RexTypeIrrelevant, dst, src);
853   emitUint8(0x0F);
854   assert(Ty == IceType_v8i16);
855   (void)Ty;
856   emitUint8(0xF5);
857   emitXmmRegisterOperand(dst, src);
858 }
859 
pmaddwd(Type Ty,XmmRegister dst,const AsmAddress & src)860 void AssemblerX8664::pmaddwd(Type Ty, XmmRegister dst, const AsmAddress &src) {
861   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
862   emitUint8(0x66);
863   emitRex(RexTypeIrrelevant, src, dst);
864   emitUint8(0x0F);
865   assert(Ty == IceType_v8i16);
866   (void)Ty;
867   emitUint8(0xF5);
868   emitOperand(gprEncoding(dst), src);
869 }
870 
pmuludq(Type,XmmRegister dst,XmmRegister src)871 void AssemblerX8664::pmuludq(Type /* Ty */, XmmRegister dst, XmmRegister src) {
872   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
873   emitUint8(0x66);
874   emitRexRB(RexTypeIrrelevant, dst, src);
875   emitUint8(0x0F);
876   emitUint8(0xF4);
877   emitXmmRegisterOperand(dst, src);
878 }
879 
pmuludq(Type,XmmRegister dst,const AsmAddress & src)880 void AssemblerX8664::pmuludq(Type /* Ty */, XmmRegister dst,
881                              const AsmAddress &src) {
882   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
883   emitUint8(0x66);
884   emitRex(RexTypeIrrelevant, src, dst);
885   emitUint8(0x0F);
886   emitUint8(0xF4);
887   emitOperand(gprEncoding(dst), src);
888 }
889 
por(Type,XmmRegister dst,XmmRegister src)890 void AssemblerX8664::por(Type /* Ty */, XmmRegister dst, XmmRegister src) {
891   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
892   emitUint8(0x66);
893   emitRexRB(RexTypeIrrelevant, dst, src);
894   emitUint8(0x0F);
895   emitUint8(0xEB);
896   emitXmmRegisterOperand(dst, src);
897 }
898 
por(Type,XmmRegister dst,const AsmAddress & src)899 void AssemblerX8664::por(Type /* Ty */, XmmRegister dst,
900                          const AsmAddress &src) {
901   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
902   emitUint8(0x66);
903   emitRex(RexTypeIrrelevant, src, dst);
904   emitUint8(0x0F);
905   emitUint8(0xEB);
906   emitOperand(gprEncoding(dst), src);
907 }
908 
psub(Type Ty,XmmRegister dst,XmmRegister src)909 void AssemblerX8664::psub(Type Ty, XmmRegister dst, XmmRegister src) {
910   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
911   emitUint8(0x66);
912   emitRexRB(RexTypeIrrelevant, dst, src);
913   emitUint8(0x0F);
914   if (isByteSizedArithType(Ty)) {
915     emitUint8(0xF8);
916   } else if (Ty == IceType_i16) {
917     emitUint8(0xF9);
918   } else {
919     emitUint8(0xFA);
920   }
921   emitXmmRegisterOperand(dst, src);
922 }
923 
psub(Type Ty,XmmRegister dst,const AsmAddress & src)924 void AssemblerX8664::psub(Type Ty, XmmRegister dst, const AsmAddress &src) {
925   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
926   emitUint8(0x66);
927   emitRex(RexTypeIrrelevant, src, dst);
928   emitUint8(0x0F);
929   if (isByteSizedArithType(Ty)) {
930     emitUint8(0xF8);
931   } else if (Ty == IceType_i16) {
932     emitUint8(0xF9);
933   } else {
934     emitUint8(0xFA);
935   }
936   emitOperand(gprEncoding(dst), src);
937 }
938 
psubs(Type Ty,XmmRegister dst,XmmRegister src)939 void AssemblerX8664::psubs(Type Ty, XmmRegister dst, XmmRegister src) {
940   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
941   emitUint8(0x66);
942   emitRexRB(RexTypeIrrelevant, dst, src);
943   emitUint8(0x0F);
944   if (isByteSizedArithType(Ty)) {
945     emitUint8(0xE8);
946   } else if (Ty == IceType_i16) {
947     emitUint8(0xE9);
948   } else {
949     assert(false && "Unexpected psubs operand type");
950   }
951   emitXmmRegisterOperand(dst, src);
952 }
953 
psubs(Type Ty,XmmRegister dst,const AsmAddress & src)954 void AssemblerX8664::psubs(Type Ty, XmmRegister dst, const AsmAddress &src) {
955   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
956   emitUint8(0x66);
957   emitRex(RexTypeIrrelevant, src, dst);
958   emitUint8(0x0F);
959   if (isByteSizedArithType(Ty)) {
960     emitUint8(0xE8);
961   } else if (Ty == IceType_i16) {
962     emitUint8(0xE9);
963   } else {
964     assert(false && "Unexpected psubs operand type");
965   }
966   emitOperand(gprEncoding(dst), src);
967 }
968 
psubus(Type Ty,XmmRegister dst,XmmRegister src)969 void AssemblerX8664::psubus(Type Ty, XmmRegister dst, XmmRegister src) {
970   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
971   emitUint8(0x66);
972   emitRexRB(RexTypeIrrelevant, dst, src);
973   emitUint8(0x0F);
974   if (isByteSizedArithType(Ty)) {
975     emitUint8(0xD8);
976   } else if (Ty == IceType_i16) {
977     emitUint8(0xD9);
978   } else {
979     assert(false && "Unexpected psubus operand type");
980   }
981   emitXmmRegisterOperand(dst, src);
982 }
983 
psubus(Type Ty,XmmRegister dst,const AsmAddress & src)984 void AssemblerX8664::psubus(Type Ty, XmmRegister dst, const AsmAddress &src) {
985   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
986   emitUint8(0x66);
987   emitRex(RexTypeIrrelevant, src, dst);
988   emitUint8(0x0F);
989   if (isByteSizedArithType(Ty)) {
990     emitUint8(0xD8);
991   } else if (Ty == IceType_i16) {
992     emitUint8(0xD9);
993   } else {
994     assert(false && "Unexpected psubus operand type");
995   }
996   emitOperand(gprEncoding(dst), src);
997 }
998 
pxor(Type,XmmRegister dst,XmmRegister src)999 void AssemblerX8664::pxor(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1000   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1001   emitUint8(0x66);
1002   emitRexRB(RexTypeIrrelevant, dst, src);
1003   emitUint8(0x0F);
1004   emitUint8(0xEF);
1005   emitXmmRegisterOperand(dst, src);
1006 }
1007 
pxor(Type,XmmRegister dst,const AsmAddress & src)1008 void AssemblerX8664::pxor(Type /* Ty */, XmmRegister dst,
1009                           const AsmAddress &src) {
1010   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1011   emitUint8(0x66);
1012   emitRex(RexTypeIrrelevant, src, dst);
1013   emitUint8(0x0F);
1014   emitUint8(0xEF);
1015   emitOperand(gprEncoding(dst), src);
1016 }
1017 
psll(Type Ty,XmmRegister dst,XmmRegister src)1018 void AssemblerX8664::psll(Type Ty, XmmRegister dst, XmmRegister src) {
1019   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1020   emitUint8(0x66);
1021   emitRexRB(RexTypeIrrelevant, dst, src);
1022   emitUint8(0x0F);
1023   if (Ty == IceType_i16) {
1024     emitUint8(0xF1);
1025   } else {
1026     assert(Ty == IceType_i32);
1027     emitUint8(0xF2);
1028   }
1029   emitXmmRegisterOperand(dst, src);
1030 }
1031 
psll(Type Ty,XmmRegister dst,const AsmAddress & src)1032 void AssemblerX8664::psll(Type Ty, XmmRegister dst, const AsmAddress &src) {
1033   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1034   emitUint8(0x66);
1035   emitRex(RexTypeIrrelevant, src, dst);
1036   emitUint8(0x0F);
1037   if (Ty == IceType_i16) {
1038     emitUint8(0xF1);
1039   } else {
1040     assert(Ty == IceType_i32);
1041     emitUint8(0xF2);
1042   }
1043   emitOperand(gprEncoding(dst), src);
1044 }
1045 
psll(Type Ty,XmmRegister dst,const Immediate & imm)1046 void AssemblerX8664::psll(Type Ty, XmmRegister dst, const Immediate &imm) {
1047   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1048   assert(imm.is_int8());
1049   emitUint8(0x66);
1050   emitRexB(RexTypeIrrelevant, dst);
1051   emitUint8(0x0F);
1052   if (Ty == IceType_i16) {
1053     emitUint8(0x71);
1054   } else {
1055     assert(Ty == IceType_i32);
1056     emitUint8(0x72);
1057   }
1058   emitRegisterOperand(6, gprEncoding(dst));
1059   emitUint8(imm.value() & 0xFF);
1060 }
1061 
psra(Type Ty,XmmRegister dst,XmmRegister src)1062 void AssemblerX8664::psra(Type Ty, XmmRegister dst, XmmRegister src) {
1063   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1064   emitUint8(0x66);
1065   emitRexRB(RexTypeIrrelevant, dst, src);
1066   emitUint8(0x0F);
1067   if (Ty == IceType_i16) {
1068     emitUint8(0xE1);
1069   } else {
1070     assert(Ty == IceType_i32);
1071     emitUint8(0xE2);
1072   }
1073   emitXmmRegisterOperand(dst, src);
1074 }
1075 
psra(Type Ty,XmmRegister dst,const AsmAddress & src)1076 void AssemblerX8664::psra(Type Ty, XmmRegister dst, const AsmAddress &src) {
1077   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1078   emitUint8(0x66);
1079   emitRex(RexTypeIrrelevant, src, dst);
1080   emitUint8(0x0F);
1081   if (Ty == IceType_i16) {
1082     emitUint8(0xE1);
1083   } else {
1084     assert(Ty == IceType_i32);
1085     emitUint8(0xE2);
1086   }
1087   emitOperand(gprEncoding(dst), src);
1088 }
1089 
psra(Type Ty,XmmRegister dst,const Immediate & imm)1090 void AssemblerX8664::psra(Type Ty, XmmRegister dst, const Immediate &imm) {
1091   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1092   assert(imm.is_int8());
1093   emitUint8(0x66);
1094   emitRexB(RexTypeIrrelevant, dst);
1095   emitUint8(0x0F);
1096   if (Ty == IceType_i16) {
1097     emitUint8(0x71);
1098   } else {
1099     assert(Ty == IceType_i32);
1100     emitUint8(0x72);
1101   }
1102   emitRegisterOperand(4, gprEncoding(dst));
1103   emitUint8(imm.value() & 0xFF);
1104 }
1105 
psrl(Type Ty,XmmRegister dst,XmmRegister src)1106 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, XmmRegister src) {
1107   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1108   emitUint8(0x66);
1109   emitRexRB(RexTypeIrrelevant, dst, src);
1110   emitUint8(0x0F);
1111   if (Ty == IceType_i16) {
1112     emitUint8(0xD1);
1113   } else if (Ty == IceType_f64) {
1114     emitUint8(0xD3);
1115   } else {
1116     assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1117     emitUint8(0xD2);
1118   }
1119   emitXmmRegisterOperand(dst, src);
1120 }
1121 
psrl(Type Ty,XmmRegister dst,const AsmAddress & src)1122 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, const AsmAddress &src) {
1123   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1124   emitUint8(0x66);
1125   emitRex(RexTypeIrrelevant, src, dst);
1126   emitUint8(0x0F);
1127   if (Ty == IceType_i16) {
1128     emitUint8(0xD1);
1129   } else if (Ty == IceType_f64) {
1130     emitUint8(0xD3);
1131   } else {
1132     assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1133     emitUint8(0xD2);
1134   }
1135   emitOperand(gprEncoding(dst), src);
1136 }
1137 
psrl(Type Ty,XmmRegister dst,const Immediate & imm)1138 void AssemblerX8664::psrl(Type Ty, XmmRegister dst, const Immediate &imm) {
1139   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1140   assert(imm.is_int8());
1141   emitUint8(0x66);
1142   emitRexB(RexTypeIrrelevant, dst);
1143   emitUint8(0x0F);
1144   if (Ty == IceType_i16) {
1145     emitUint8(0x71);
1146   } else if (Ty == IceType_f64) {
1147     emitUint8(0x73);
1148   } else {
1149     assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1150     emitUint8(0x72);
1151   }
1152   emitRegisterOperand(2, gprEncoding(dst));
1153   emitUint8(imm.value() & 0xFF);
1154 }
1155 
1156 // {add,sub,mul,div}ps are given a Ty parameter for consistency with
1157 // {add,sub,mul,div}ss. In the future, when the PNaCl ABI allows addpd, etc.,
1158 // we can use the Ty parameter to decide on adding a 0x66 prefix.
1159 
addps(Type,XmmRegister dst,XmmRegister src)1160 void AssemblerX8664::addps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1161   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1162   emitRexRB(RexTypeIrrelevant, dst, src);
1163   emitUint8(0x0F);
1164   emitUint8(0x58);
1165   emitXmmRegisterOperand(dst, src);
1166 }
1167 
addps(Type,XmmRegister dst,const AsmAddress & src)1168 void AssemblerX8664::addps(Type /* Ty */, XmmRegister dst,
1169                            const AsmAddress &src) {
1170   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1171   emitRex(RexTypeIrrelevant, src, dst);
1172   emitUint8(0x0F);
1173   emitUint8(0x58);
1174   emitOperand(gprEncoding(dst), src);
1175 }
1176 
subps(Type,XmmRegister dst,XmmRegister src)1177 void AssemblerX8664::subps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1178   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1179   emitRexRB(RexTypeIrrelevant, dst, src);
1180   emitUint8(0x0F);
1181   emitUint8(0x5C);
1182   emitXmmRegisterOperand(dst, src);
1183 }
1184 
subps(Type,XmmRegister dst,const AsmAddress & src)1185 void AssemblerX8664::subps(Type /* Ty */, XmmRegister dst,
1186                            const AsmAddress &src) {
1187   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1188   emitRex(RexTypeIrrelevant, src, dst);
1189   emitUint8(0x0F);
1190   emitUint8(0x5C);
1191   emitOperand(gprEncoding(dst), src);
1192 }
1193 
divps(Type,XmmRegister dst,XmmRegister src)1194 void AssemblerX8664::divps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1195   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1196   emitRexRB(RexTypeIrrelevant, dst, src);
1197   emitUint8(0x0F);
1198   emitUint8(0x5E);
1199   emitXmmRegisterOperand(dst, src);
1200 }
1201 
divps(Type,XmmRegister dst,const AsmAddress & src)1202 void AssemblerX8664::divps(Type /* Ty */, XmmRegister dst,
1203                            const AsmAddress &src) {
1204   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1205   emitRex(RexTypeIrrelevant, src, dst);
1206   emitUint8(0x0F);
1207   emitUint8(0x5E);
1208   emitOperand(gprEncoding(dst), src);
1209 }
1210 
mulps(Type,XmmRegister dst,XmmRegister src)1211 void AssemblerX8664::mulps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1212   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1213   emitRexRB(RexTypeIrrelevant, dst, src);
1214   emitUint8(0x0F);
1215   emitUint8(0x59);
1216   emitXmmRegisterOperand(dst, src);
1217 }
1218 
mulps(Type,XmmRegister dst,const AsmAddress & src)1219 void AssemblerX8664::mulps(Type /* Ty */, XmmRegister dst,
1220                            const AsmAddress &src) {
1221   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1222   emitRex(RexTypeIrrelevant, src, dst);
1223   emitUint8(0x0F);
1224   emitUint8(0x59);
1225   emitOperand(gprEncoding(dst), src);
1226 }
1227 
minps(Type Ty,XmmRegister dst,XmmRegister src)1228 void AssemblerX8664::minps(Type Ty, XmmRegister dst, XmmRegister src) {
1229   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1230   if (!isFloat32Asserting32Or64(Ty))
1231     emitUint8(0x66);
1232   emitRexRB(RexTypeIrrelevant, dst, src);
1233   emitUint8(0x0F);
1234   emitUint8(0x5D);
1235   emitXmmRegisterOperand(dst, src);
1236 }
1237 
minps(Type Ty,XmmRegister dst,const AsmAddress & src)1238 void AssemblerX8664::minps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1239   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1240   if (!isFloat32Asserting32Or64(Ty))
1241     emitUint8(0x66);
1242   emitRex(RexTypeIrrelevant, src, dst);
1243   emitUint8(0x0F);
1244   emitUint8(0x5D);
1245   emitOperand(gprEncoding(dst), src);
1246 }
1247 
minss(Type Ty,XmmRegister dst,XmmRegister src)1248 void AssemblerX8664::minss(Type Ty, XmmRegister dst, XmmRegister src) {
1249   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1250   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1251   emitRexRB(RexTypeIrrelevant, dst, src);
1252   emitUint8(0x0F);
1253   emitUint8(0x5D);
1254   emitXmmRegisterOperand(dst, src);
1255 }
1256 
minss(Type Ty,XmmRegister dst,const AsmAddress & src)1257 void AssemblerX8664::minss(Type Ty, XmmRegister dst, const AsmAddress &src) {
1258   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1259   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1260   emitRex(RexTypeIrrelevant, src, dst);
1261   emitUint8(0x0F);
1262   emitUint8(0x5D);
1263   emitOperand(gprEncoding(dst), src);
1264 }
1265 
maxps(Type Ty,XmmRegister dst,XmmRegister src)1266 void AssemblerX8664::maxps(Type Ty, XmmRegister dst, XmmRegister src) {
1267   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1268   if (!isFloat32Asserting32Or64(Ty))
1269     emitUint8(0x66);
1270   emitRexRB(RexTypeIrrelevant, dst, src);
1271   emitUint8(0x0F);
1272   emitUint8(0x5F);
1273   emitXmmRegisterOperand(dst, src);
1274 }
1275 
maxps(Type Ty,XmmRegister dst,const AsmAddress & src)1276 void AssemblerX8664::maxps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1277   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1278   if (!isFloat32Asserting32Or64(Ty))
1279     emitUint8(0x66);
1280   emitRex(RexTypeIrrelevant, src, dst);
1281   emitUint8(0x0F);
1282   emitUint8(0x5F);
1283   emitOperand(gprEncoding(dst), src);
1284 }
1285 
maxss(Type Ty,XmmRegister dst,XmmRegister src)1286 void AssemblerX8664::maxss(Type Ty, XmmRegister dst, XmmRegister src) {
1287   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1288   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1289   emitRexRB(RexTypeIrrelevant, dst, src);
1290   emitUint8(0x0F);
1291   emitUint8(0x5F);
1292   emitXmmRegisterOperand(dst, src);
1293 }
1294 
maxss(Type Ty,XmmRegister dst,const AsmAddress & src)1295 void AssemblerX8664::maxss(Type Ty, XmmRegister dst, const AsmAddress &src) {
1296   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1297   emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1298   emitRex(RexTypeIrrelevant, src, dst);
1299   emitUint8(0x0F);
1300   emitUint8(0x5F);
1301   emitOperand(gprEncoding(dst), src);
1302 }
1303 
andnps(Type Ty,XmmRegister dst,XmmRegister src)1304 void AssemblerX8664::andnps(Type Ty, XmmRegister dst, XmmRegister src) {
1305   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1306   if (!isFloat32Asserting32Or64(Ty))
1307     emitUint8(0x66);
1308   emitRexRB(RexTypeIrrelevant, dst, src);
1309   emitUint8(0x0F);
1310   emitUint8(0x55);
1311   emitXmmRegisterOperand(dst, src);
1312 }
1313 
andnps(Type Ty,XmmRegister dst,const AsmAddress & src)1314 void AssemblerX8664::andnps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1315   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1316   if (!isFloat32Asserting32Or64(Ty))
1317     emitUint8(0x66);
1318   emitRex(RexTypeIrrelevant, src, dst);
1319   emitUint8(0x0F);
1320   emitUint8(0x55);
1321   emitOperand(gprEncoding(dst), src);
1322 }
1323 
andps(Type Ty,XmmRegister dst,XmmRegister src)1324 void AssemblerX8664::andps(Type Ty, XmmRegister dst, XmmRegister src) {
1325   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1326   if (!isFloat32Asserting32Or64(Ty))
1327     emitUint8(0x66);
1328   emitRexRB(RexTypeIrrelevant, dst, src);
1329   emitUint8(0x0F);
1330   emitUint8(0x54);
1331   emitXmmRegisterOperand(dst, src);
1332 }
1333 
andps(Type Ty,XmmRegister dst,const AsmAddress & src)1334 void AssemblerX8664::andps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1335   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1336   if (!isFloat32Asserting32Or64(Ty))
1337     emitUint8(0x66);
1338   emitRex(RexTypeIrrelevant, src, dst);
1339   emitUint8(0x0F);
1340   emitUint8(0x54);
1341   emitOperand(gprEncoding(dst), src);
1342 }
1343 
orps(Type Ty,XmmRegister dst,XmmRegister src)1344 void AssemblerX8664::orps(Type Ty, XmmRegister dst, XmmRegister src) {
1345   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1346   if (!isFloat32Asserting32Or64(Ty))
1347     emitUint8(0x66);
1348   emitRexRB(RexTypeIrrelevant, dst, src);
1349   emitUint8(0x0F);
1350   emitUint8(0x56);
1351   emitXmmRegisterOperand(dst, src);
1352 }
1353 
orps(Type Ty,XmmRegister dst,const AsmAddress & src)1354 void AssemblerX8664::orps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1355   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1356   if (!isFloat32Asserting32Or64(Ty))
1357     emitUint8(0x66);
1358   emitRex(RexTypeIrrelevant, src, dst);
1359   emitUint8(0x0F);
1360   emitUint8(0x56);
1361   emitOperand(gprEncoding(dst), src);
1362 }
1363 
blendvps(Type,XmmRegister dst,XmmRegister src)1364 void AssemblerX8664::blendvps(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1365   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1366   emitUint8(0x66);
1367   emitRexRB(RexTypeIrrelevant, dst, src);
1368   emitUint8(0x0F);
1369   emitUint8(0x38);
1370   emitUint8(0x14);
1371   emitXmmRegisterOperand(dst, src);
1372 }
1373 
blendvps(Type,XmmRegister dst,const AsmAddress & src)1374 void AssemblerX8664::blendvps(Type /* Ty */, XmmRegister dst,
1375                               const AsmAddress &src) {
1376   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1377   emitUint8(0x66);
1378   emitRex(RexTypeIrrelevant, src, dst);
1379   emitUint8(0x0F);
1380   emitUint8(0x38);
1381   emitUint8(0x14);
1382   emitOperand(gprEncoding(dst), src);
1383 }
1384 
pblendvb(Type,XmmRegister dst,XmmRegister src)1385 void AssemblerX8664::pblendvb(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1386   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1387   emitUint8(0x66);
1388   emitRexRB(RexTypeIrrelevant, dst, src);
1389   emitUint8(0x0F);
1390   emitUint8(0x38);
1391   emitUint8(0x10);
1392   emitXmmRegisterOperand(dst, src);
1393 }
1394 
pblendvb(Type,XmmRegister dst,const AsmAddress & src)1395 void AssemblerX8664::pblendvb(Type /* Ty */, XmmRegister dst,
1396                               const AsmAddress &src) {
1397   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1398   emitUint8(0x66);
1399   emitRex(RexTypeIrrelevant, src, dst);
1400   emitUint8(0x0F);
1401   emitUint8(0x38);
1402   emitUint8(0x10);
1403   emitOperand(gprEncoding(dst), src);
1404 }
1405 
cmpps(Type Ty,XmmRegister dst,XmmRegister src,CmppsCond CmpCondition)1406 void AssemblerX8664::cmpps(Type Ty, XmmRegister dst, XmmRegister src,
1407                            CmppsCond CmpCondition) {
1408   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1409   if (Ty == IceType_f64)
1410     emitUint8(0x66);
1411   emitRexRB(RexTypeIrrelevant, dst, src);
1412   emitUint8(0x0F);
1413   emitUint8(0xC2);
1414   emitXmmRegisterOperand(dst, src);
1415   emitUint8(CmpCondition);
1416 }
1417 
cmpps(Type Ty,XmmRegister dst,const AsmAddress & src,CmppsCond CmpCondition)1418 void AssemblerX8664::cmpps(Type Ty, XmmRegister dst, const AsmAddress &src,
1419                            CmppsCond CmpCondition) {
1420   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1421   if (Ty == IceType_f64)
1422     emitUint8(0x66);
1423   emitRex(RexTypeIrrelevant, src, dst);
1424   emitUint8(0x0F);
1425   emitUint8(0xC2);
1426   static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1427   emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1428   emitUint8(CmpCondition);
1429 }
1430 
sqrtps(XmmRegister dst)1431 void AssemblerX8664::sqrtps(XmmRegister dst) {
1432   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1433   emitRexRB(RexTypeIrrelevant, dst, dst);
1434   emitUint8(0x0F);
1435   emitUint8(0x51);
1436   emitXmmRegisterOperand(dst, dst);
1437 }
1438 
rsqrtps(XmmRegister dst)1439 void AssemblerX8664::rsqrtps(XmmRegister dst) {
1440   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1441   emitRexRB(RexTypeIrrelevant, dst, dst);
1442   emitUint8(0x0F);
1443   emitUint8(0x52);
1444   emitXmmRegisterOperand(dst, dst);
1445 }
1446 
reciprocalps(XmmRegister dst)1447 void AssemblerX8664::reciprocalps(XmmRegister dst) {
1448   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1449   emitRexRB(RexTypeIrrelevant, dst, dst);
1450   emitUint8(0x0F);
1451   emitUint8(0x53);
1452   emitXmmRegisterOperand(dst, dst);
1453 }
1454 
movhlps(XmmRegister dst,XmmRegister src)1455 void AssemblerX8664::movhlps(XmmRegister dst, XmmRegister src) {
1456   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1457   emitRexRB(RexTypeIrrelevant, dst, src);
1458   emitUint8(0x0F);
1459   emitUint8(0x12);
1460   emitXmmRegisterOperand(dst, src);
1461 }
1462 
movlhps(XmmRegister dst,XmmRegister src)1463 void AssemblerX8664::movlhps(XmmRegister dst, XmmRegister src) {
1464   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1465   emitRexRB(RexTypeIrrelevant, dst, src);
1466   emitUint8(0x0F);
1467   emitUint8(0x16);
1468   emitXmmRegisterOperand(dst, src);
1469 }
1470 
unpcklps(XmmRegister dst,XmmRegister src)1471 void AssemblerX8664::unpcklps(XmmRegister dst, XmmRegister src) {
1472   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1473   emitRexRB(RexTypeIrrelevant, dst, src);
1474   emitUint8(0x0F);
1475   emitUint8(0x14);
1476   emitXmmRegisterOperand(dst, src);
1477 }
1478 
unpckhps(XmmRegister dst,XmmRegister src)1479 void AssemblerX8664::unpckhps(XmmRegister dst, XmmRegister src) {
1480   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1481   emitRexRB(RexTypeIrrelevant, dst, src);
1482   emitUint8(0x0F);
1483   emitUint8(0x15);
1484   emitXmmRegisterOperand(dst, src);
1485 }
1486 
unpcklpd(XmmRegister dst,XmmRegister src)1487 void AssemblerX8664::unpcklpd(XmmRegister dst, XmmRegister src) {
1488   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1489   emitUint8(0x66);
1490   emitRexRB(RexTypeIrrelevant, dst, src);
1491   emitUint8(0x0F);
1492   emitUint8(0x14);
1493   emitXmmRegisterOperand(dst, src);
1494 }
1495 
unpckhpd(XmmRegister dst,XmmRegister src)1496 void AssemblerX8664::unpckhpd(XmmRegister dst, XmmRegister src) {
1497   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1498   emitUint8(0x66);
1499   emitRexRB(RexTypeIrrelevant, dst, src);
1500   emitUint8(0x0F);
1501   emitUint8(0x15);
1502   emitXmmRegisterOperand(dst, src);
1503 }
1504 
set1ps(XmmRegister dst,GPRRegister tmp1,const Immediate & imm)1505 void AssemblerX8664::set1ps(XmmRegister dst, GPRRegister tmp1,
1506                             const Immediate &imm) {
1507   // Load 32-bit immediate value into tmp1.
1508   mov(IceType_i32, tmp1, imm);
1509   // Move value from tmp1 into dst.
1510   movd(IceType_i32, dst, tmp1);
1511   // Broadcast low lane into other three lanes.
1512   shufps(RexTypeIrrelevant, dst, dst, Immediate(0x0));
1513 }
1514 
pshufb(Type,XmmRegister dst,XmmRegister src)1515 void AssemblerX8664::pshufb(Type /* Ty */, XmmRegister dst, XmmRegister src) {
1516   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1517   emitUint8(0x66);
1518   emitRexRB(RexTypeIrrelevant, dst, src);
1519   emitUint8(0x0F);
1520   emitUint8(0x38);
1521   emitUint8(0x00);
1522   emitXmmRegisterOperand(dst, src);
1523 }
1524 
pshufb(Type,XmmRegister dst,const AsmAddress & src)1525 void AssemblerX8664::pshufb(Type /* Ty */, XmmRegister dst,
1526                             const AsmAddress &src) {
1527   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1528   emitUint8(0x66);
1529   emitRex(RexTypeIrrelevant, src, dst);
1530   emitUint8(0x0F);
1531   emitUint8(0x38);
1532   emitUint8(0x00);
1533   emitOperand(gprEncoding(dst), src);
1534 }
1535 
pshufd(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1536 void AssemblerX8664::pshufd(Type /* Ty */, XmmRegister dst, XmmRegister src,
1537                             const Immediate &imm) {
1538   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1539   emitUint8(0x66);
1540   emitRexRB(RexTypeIrrelevant, dst, src);
1541   emitUint8(0x0F);
1542   emitUint8(0x70);
1543   emitXmmRegisterOperand(dst, src);
1544   assert(imm.is_uint8());
1545   emitUint8(imm.value());
1546 }
1547 
pshufd(Type,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1548 void AssemblerX8664::pshufd(Type /* Ty */, XmmRegister dst,
1549                             const AsmAddress &src, const Immediate &imm) {
1550   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1551   emitUint8(0x66);
1552   emitRex(RexTypeIrrelevant, src, dst);
1553   emitUint8(0x0F);
1554   emitUint8(0x70);
1555   static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1556   emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1557   assert(imm.is_uint8());
1558   emitUint8(imm.value());
1559 }
1560 
punpckl(Type Ty,XmmRegister Dst,XmmRegister Src)1561 void AssemblerX8664::punpckl(Type Ty, XmmRegister Dst, XmmRegister Src) {
1562   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1563   emitUint8(0x66);
1564   emitRexRB(RexTypeIrrelevant, Dst, Src);
1565   emitUint8(0x0F);
1566   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1567     emitUint8(0x62);
1568   } else if (Ty == IceType_v8i16) {
1569     emitUint8(0x61);
1570   } else if (Ty == IceType_v16i8) {
1571     emitUint8(0x60);
1572   } else {
1573     assert(false && "Unexpected vector unpack operand type");
1574   }
1575   emitXmmRegisterOperand(Dst, Src);
1576 }
1577 
punpckl(Type Ty,XmmRegister Dst,const AsmAddress & Src)1578 void AssemblerX8664::punpckl(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1579   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1580   emitUint8(0x66);
1581   emitRex(RexTypeIrrelevant, Src, Dst);
1582   emitUint8(0x0F);
1583   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1584     emitUint8(0x62);
1585   } else if (Ty == IceType_v8i16) {
1586     emitUint8(0x61);
1587   } else if (Ty == IceType_v16i8) {
1588     emitUint8(0x60);
1589   } else {
1590     assert(false && "Unexpected vector unpack operand type");
1591   }
1592   emitOperand(gprEncoding(Dst), Src);
1593 }
1594 
punpckh(Type Ty,XmmRegister Dst,XmmRegister Src)1595 void AssemblerX8664::punpckh(Type Ty, XmmRegister Dst, XmmRegister Src) {
1596   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1597   emitUint8(0x66);
1598   emitRexRB(RexTypeIrrelevant, Dst, Src);
1599   emitUint8(0x0F);
1600   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1601     emitUint8(0x6A);
1602   } else if (Ty == IceType_v8i16) {
1603     emitUint8(0x69);
1604   } else if (Ty == IceType_v16i8) {
1605     emitUint8(0x68);
1606   } else {
1607     assert(false && "Unexpected vector unpack operand type");
1608   }
1609   emitXmmRegisterOperand(Dst, Src);
1610 }
1611 
punpckh(Type Ty,XmmRegister Dst,const AsmAddress & Src)1612 void AssemblerX8664::punpckh(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1613   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1614   emitUint8(0x66);
1615   emitRex(RexTypeIrrelevant, Src, Dst);
1616   emitUint8(0x0F);
1617   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1618     emitUint8(0x6A);
1619   } else if (Ty == IceType_v8i16) {
1620     emitUint8(0x69);
1621   } else if (Ty == IceType_v16i8) {
1622     emitUint8(0x68);
1623   } else {
1624     assert(false && "Unexpected vector unpack operand type");
1625   }
1626   emitOperand(gprEncoding(Dst), Src);
1627 }
1628 
packss(Type Ty,XmmRegister Dst,XmmRegister Src)1629 void AssemblerX8664::packss(Type Ty, XmmRegister Dst, XmmRegister Src) {
1630   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1631   emitUint8(0x66);
1632   emitRexRB(RexTypeIrrelevant, Dst, Src);
1633   emitUint8(0x0F);
1634   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1635     emitUint8(0x6B);
1636   } else if (Ty == IceType_v8i16) {
1637     emitUint8(0x63);
1638   } else {
1639     assert(false && "Unexpected vector pack operand type");
1640   }
1641   emitXmmRegisterOperand(Dst, Src);
1642 }
1643 
packss(Type Ty,XmmRegister Dst,const AsmAddress & Src)1644 void AssemblerX8664::packss(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1645   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1646   emitUint8(0x66);
1647   emitRex(RexTypeIrrelevant, Src, Dst);
1648   emitUint8(0x0F);
1649   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1650     emitUint8(0x6B);
1651   } else if (Ty == IceType_v8i16) {
1652     emitUint8(0x63);
1653   } else {
1654     assert(false && "Unexpected vector pack operand type");
1655   }
1656   emitOperand(gprEncoding(Dst), Src);
1657 }
1658 
packus(Type Ty,XmmRegister Dst,XmmRegister Src)1659 void AssemblerX8664::packus(Type Ty, XmmRegister Dst, XmmRegister Src) {
1660   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1661   emitUint8(0x66);
1662   emitRexRB(RexTypeIrrelevant, Dst, Src);
1663   emitUint8(0x0F);
1664   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1665     emitUint8(0x38);
1666     emitUint8(0x2B);
1667   } else if (Ty == IceType_v8i16) {
1668     emitUint8(0x67);
1669   } else {
1670     assert(false && "Unexpected vector pack operand type");
1671   }
1672   emitXmmRegisterOperand(Dst, Src);
1673 }
1674 
packus(Type Ty,XmmRegister Dst,const AsmAddress & Src)1675 void AssemblerX8664::packus(Type Ty, XmmRegister Dst, const AsmAddress &Src) {
1676   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1677   emitUint8(0x66);
1678   emitRex(RexTypeIrrelevant, Src, Dst);
1679   emitUint8(0x0F);
1680   if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1681     emitUint8(0x38);
1682     emitUint8(0x2B);
1683   } else if (Ty == IceType_v8i16) {
1684     emitUint8(0x67);
1685   } else {
1686     assert(false && "Unexpected vector pack operand type");
1687   }
1688   emitOperand(gprEncoding(Dst), Src);
1689 }
1690 
shufps(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1691 void AssemblerX8664::shufps(Type /* Ty */, XmmRegister dst, XmmRegister src,
1692                             const Immediate &imm) {
1693   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1694   emitRexRB(RexTypeIrrelevant, dst, src);
1695   emitUint8(0x0F);
1696   emitUint8(0xC6);
1697   emitXmmRegisterOperand(dst, src);
1698   assert(imm.is_uint8());
1699   emitUint8(imm.value());
1700 }
1701 
shufps(Type,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1702 void AssemblerX8664::shufps(Type /* Ty */, XmmRegister dst,
1703                             const AsmAddress &src, const Immediate &imm) {
1704   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1705   emitRex(RexTypeIrrelevant, src, dst);
1706   emitUint8(0x0F);
1707   emitUint8(0xC6);
1708   static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1709   emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1710   assert(imm.is_uint8());
1711   emitUint8(imm.value());
1712 }
1713 
sqrtpd(XmmRegister dst)1714 void AssemblerX8664::sqrtpd(XmmRegister dst) {
1715   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1716   emitUint8(0x66);
1717   emitRexRB(RexTypeIrrelevant, dst, dst);
1718   emitUint8(0x0F);
1719   emitUint8(0x51);
1720   emitXmmRegisterOperand(dst, dst);
1721 }
1722 
cvtdq2ps(Type,XmmRegister dst,XmmRegister src)1723 void AssemblerX8664::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
1724                               XmmRegister src) {
1725   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1726   emitRexRB(RexTypeIrrelevant, dst, src);
1727   emitUint8(0x0F);
1728   emitUint8(0x5B);
1729   emitXmmRegisterOperand(dst, src);
1730 }
1731 
cvtdq2ps(Type,XmmRegister dst,const AsmAddress & src)1732 void AssemblerX8664::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
1733                               const AsmAddress &src) {
1734   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1735   emitRex(RexTypeIrrelevant, src, dst);
1736   emitUint8(0x0F);
1737   emitUint8(0x5B);
1738   emitOperand(gprEncoding(dst), src);
1739 }
1740 
cvttps2dq(Type,XmmRegister dst,XmmRegister src)1741 void AssemblerX8664::cvttps2dq(Type /* Ignore */, XmmRegister dst,
1742                                XmmRegister src) {
1743   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1744   emitUint8(0xF3);
1745   emitRexRB(RexTypeIrrelevant, dst, src);
1746   emitUint8(0x0F);
1747   emitUint8(0x5B);
1748   emitXmmRegisterOperand(dst, src);
1749 }
1750 
cvttps2dq(Type,XmmRegister dst,const AsmAddress & src)1751 void AssemblerX8664::cvttps2dq(Type /* Ignore */, XmmRegister dst,
1752                                const AsmAddress &src) {
1753   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1754   emitUint8(0xF3);
1755   emitRex(RexTypeIrrelevant, src, dst);
1756   emitUint8(0x0F);
1757   emitUint8(0x5B);
1758   emitOperand(gprEncoding(dst), src);
1759 }
1760 
cvtps2dq(Type,XmmRegister dst,XmmRegister src)1761 void AssemblerX8664::cvtps2dq(Type /* Ignore */, XmmRegister dst,
1762                               XmmRegister src) {
1763   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1764   emitUint8(0x66);
1765   emitRexRB(RexTypeIrrelevant, dst, src);
1766   emitUint8(0x0F);
1767   emitUint8(0x5B);
1768   emitXmmRegisterOperand(dst, src);
1769 }
1770 
cvtps2dq(Type,XmmRegister dst,const AsmAddress & src)1771 void AssemblerX8664::cvtps2dq(Type /* Ignore */, XmmRegister dst,
1772                               const AsmAddress &src) {
1773   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1774   emitUint8(0x66);
1775   emitRex(RexTypeIrrelevant, src, dst);
1776   emitUint8(0x0F);
1777   emitUint8(0x5B);
1778   emitOperand(gprEncoding(dst), src);
1779 }
1780 
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,GPRRegister src)1781 void AssemblerX8664::cvtsi2ss(Type DestTy, XmmRegister dst, Type SrcTy,
1782                               GPRRegister src) {
1783   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1784   emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
1785   emitRexRB(SrcTy, dst, src);
1786   emitUint8(0x0F);
1787   emitUint8(0x2A);
1788   emitXmmRegisterOperand(dst, src);
1789 }
1790 
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,const AsmAddress & src)1791 void AssemblerX8664::cvtsi2ss(Type DestTy, XmmRegister dst, Type SrcTy,
1792                               const AsmAddress &src) {
1793   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1794   emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
1795   emitRex(SrcTy, src, dst);
1796   emitUint8(0x0F);
1797   emitUint8(0x2A);
1798   emitOperand(gprEncoding(dst), src);
1799 }
1800 
cvtfloat2float(Type SrcTy,XmmRegister dst,XmmRegister src)1801 void AssemblerX8664::cvtfloat2float(Type SrcTy, XmmRegister dst,
1802                                     XmmRegister src) {
1803   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1804   // ss2sd or sd2ss
1805   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1806   emitRexRB(RexTypeIrrelevant, dst, src);
1807   emitUint8(0x0F);
1808   emitUint8(0x5A);
1809   emitXmmRegisterOperand(dst, src);
1810 }
1811 
cvtfloat2float(Type SrcTy,XmmRegister dst,const AsmAddress & src)1812 void AssemblerX8664::cvtfloat2float(Type SrcTy, XmmRegister dst,
1813                                     const AsmAddress &src) {
1814   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1815   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1816   emitRex(RexTypeIrrelevant, src, dst);
1817   emitUint8(0x0F);
1818   emitUint8(0x5A);
1819   emitOperand(gprEncoding(dst), src);
1820 }
1821 
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)1822 void AssemblerX8664::cvttss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1823                                XmmRegister src) {
1824   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1825   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1826   emitRexRB(DestTy, dst, src);
1827   emitUint8(0x0F);
1828   emitUint8(0x2C);
1829   emitXmmRegisterOperand(dst, src);
1830 }
1831 
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,const AsmAddress & src)1832 void AssemblerX8664::cvttss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1833                                const AsmAddress &src) {
1834   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1835   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1836   emitRex(DestTy, src, dst);
1837   emitUint8(0x0F);
1838   emitUint8(0x2C);
1839   emitOperand(gprEncoding(dst), src);
1840 }
1841 
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)1842 void AssemblerX8664::cvtss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1843                               XmmRegister src) {
1844   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1845   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1846   emitRexRB(DestTy, dst, src);
1847   emitUint8(0x0F);
1848   emitUint8(0x2D);
1849   emitXmmRegisterOperand(dst, src);
1850 }
1851 
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,const AsmAddress & src)1852 void AssemblerX8664::cvtss2si(Type DestTy, GPRRegister dst, Type SrcTy,
1853                               const AsmAddress &src) {
1854   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1855   emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
1856   emitRex(DestTy, src, dst);
1857   emitUint8(0x0F);
1858   emitUint8(0x2D);
1859   emitOperand(gprEncoding(dst), src);
1860 }
1861 
ucomiss(Type Ty,XmmRegister a,XmmRegister b)1862 void AssemblerX8664::ucomiss(Type Ty, XmmRegister a, XmmRegister b) {
1863   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1864   if (Ty == IceType_f64)
1865     emitUint8(0x66);
1866   emitRexRB(RexTypeIrrelevant, a, b);
1867   emitUint8(0x0F);
1868   emitUint8(0x2E);
1869   emitXmmRegisterOperand(a, b);
1870 }
1871 
ucomiss(Type Ty,XmmRegister a,const AsmAddress & b)1872 void AssemblerX8664::ucomiss(Type Ty, XmmRegister a, const AsmAddress &b) {
1873   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1874   if (Ty == IceType_f64)
1875     emitUint8(0x66);
1876   emitRex(RexTypeIrrelevant, b, a);
1877   emitUint8(0x0F);
1878   emitUint8(0x2E);
1879   emitOperand(gprEncoding(a), b);
1880 }
1881 
movmsk(Type Ty,GPRRegister dst,XmmRegister src)1882 void AssemblerX8664::movmsk(Type Ty, GPRRegister dst, XmmRegister src) {
1883   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1884   if (Ty == IceType_v16i8) {
1885     emitUint8(0x66);
1886   } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
1887     // No operand size prefix
1888   } else {
1889     assert(false && "Unexpected movmsk operand type");
1890   }
1891   emitRexRB(RexTypeIrrelevant, dst, src);
1892   emitUint8(0x0F);
1893   if (Ty == IceType_v16i8) {
1894     emitUint8(0xD7);
1895   } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
1896     emitUint8(0x50);
1897   } else {
1898     assert(false && "Unexpected movmsk operand type");
1899   }
1900   emitXmmRegisterOperand(dst, src);
1901 }
1902 
sqrt(Type Ty,XmmRegister dst,const AsmAddress & src)1903 void AssemblerX8664::sqrt(Type Ty, XmmRegister dst, const AsmAddress &src) {
1904   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1905   if (isScalarFloatingType(Ty))
1906     emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1907   emitRex(RexTypeIrrelevant, src, dst);
1908   emitUint8(0x0F);
1909   emitUint8(0x51);
1910   emitOperand(gprEncoding(dst), src);
1911 }
1912 
sqrt(Type Ty,XmmRegister dst,XmmRegister src)1913 void AssemblerX8664::sqrt(Type Ty, XmmRegister dst, XmmRegister src) {
1914   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1915   if (isScalarFloatingType(Ty))
1916     emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1917   emitRexRB(RexTypeIrrelevant, dst, src);
1918   emitUint8(0x0F);
1919   emitUint8(0x51);
1920   emitXmmRegisterOperand(dst, src);
1921 }
1922 
xorps(Type Ty,XmmRegister dst,const AsmAddress & src)1923 void AssemblerX8664::xorps(Type Ty, XmmRegister dst, const AsmAddress &src) {
1924   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1925   if (!isFloat32Asserting32Or64(Ty))
1926     emitUint8(0x66);
1927   emitRex(RexTypeIrrelevant, src, dst);
1928   emitUint8(0x0F);
1929   emitUint8(0x57);
1930   emitOperand(gprEncoding(dst), src);
1931 }
1932 
xorps(Type Ty,XmmRegister dst,XmmRegister src)1933 void AssemblerX8664::xorps(Type Ty, XmmRegister dst, XmmRegister src) {
1934   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1935   if (!isFloat32Asserting32Or64(Ty))
1936     emitUint8(0x66);
1937   emitRexRB(RexTypeIrrelevant, dst, src);
1938   emitUint8(0x0F);
1939   emitUint8(0x57);
1940   emitXmmRegisterOperand(dst, src);
1941 }
1942 
insertps(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & imm)1943 void AssemblerX8664::insertps(Type Ty, XmmRegister dst, XmmRegister src,
1944                               const Immediate &imm) {
1945   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1946   assert(imm.is_uint8());
1947   assert(isVectorFloatingType(Ty));
1948   (void)Ty;
1949   emitUint8(0x66);
1950   emitRexRB(RexTypeIrrelevant, dst, src);
1951   emitUint8(0x0F);
1952   emitUint8(0x3A);
1953   emitUint8(0x21);
1954   emitXmmRegisterOperand(dst, src);
1955   emitUint8(imm.value());
1956 }
1957 
insertps(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1958 void AssemblerX8664::insertps(Type Ty, XmmRegister dst, const AsmAddress &src,
1959                               const Immediate &imm) {
1960   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1961   assert(imm.is_uint8());
1962   assert(isVectorFloatingType(Ty));
1963   (void)Ty;
1964   emitUint8(0x66);
1965   emitRex(RexTypeIrrelevant, src, dst);
1966   emitUint8(0x0F);
1967   emitUint8(0x3A);
1968   emitUint8(0x21);
1969   static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1970   emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1971   emitUint8(imm.value());
1972 }
1973 
pinsr(Type Ty,XmmRegister dst,GPRRegister src,const Immediate & imm)1974 void AssemblerX8664::pinsr(Type Ty, XmmRegister dst, GPRRegister src,
1975                            const Immediate &imm) {
1976   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1977   assert(imm.is_uint8());
1978   emitUint8(0x66);
1979   emitRexRB(Ty, dst, src);
1980   emitUint8(0x0F);
1981   if (Ty == IceType_i16) {
1982     emitUint8(0xC4);
1983   } else {
1984     emitUint8(0x3A);
1985     emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
1986   }
1987   emitXmmRegisterOperand(dst, src);
1988   emitUint8(imm.value());
1989 }
1990 
pinsr(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & imm)1991 void AssemblerX8664::pinsr(Type Ty, XmmRegister dst, const AsmAddress &src,
1992                            const Immediate &imm) {
1993   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1994   assert(imm.is_uint8());
1995   emitUint8(0x66);
1996   emitRex(RexTypeIrrelevant, src, dst);
1997   emitUint8(0x0F);
1998   if (Ty == IceType_i16) {
1999     emitUint8(0xC4);
2000   } else {
2001     emitUint8(0x3A);
2002     emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
2003   }
2004   static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2005   emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
2006   emitUint8(imm.value());
2007 }
2008 
pextr(Type Ty,GPRRegister dst,XmmRegister src,const Immediate & imm)2009 void AssemblerX8664::pextr(Type Ty, GPRRegister dst, XmmRegister src,
2010                            const Immediate &imm) {
2011   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2012   assert(imm.is_uint8());
2013   if (Ty == IceType_i16) {
2014     emitUint8(0x66);
2015     emitRexRB(Ty, dst, src);
2016     emitUint8(0x0F);
2017     emitUint8(0xC5);
2018     emitXmmRegisterOperand(dst, src);
2019     emitUint8(imm.value());
2020   } else {
2021     emitUint8(0x66);
2022     emitRexRB(Ty, src, dst);
2023     emitUint8(0x0F);
2024     emitUint8(0x3A);
2025     emitUint8(isByteSizedType(Ty) ? 0x14 : 0x16);
2026     // SSE 4.1 versions are "MRI" because dst can be mem, while pextrw (SSE2)
2027     // is RMI because dst must be reg.
2028     emitXmmRegisterOperand(src, dst);
2029     emitUint8(imm.value());
2030   }
2031 }
2032 
pmovsxdq(XmmRegister dst,XmmRegister src)2033 void AssemblerX8664::pmovsxdq(XmmRegister dst, XmmRegister src) {
2034   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2035   emitUint8(0x66);
2036   emitRexRB(RexTypeIrrelevant, dst, src);
2037   emitUint8(0x0F);
2038   emitUint8(0x38);
2039   emitUint8(0x25);
2040   emitXmmRegisterOperand(dst, src);
2041 }
2042 
pcmpeq(Type Ty,XmmRegister dst,XmmRegister src)2043 void AssemblerX8664::pcmpeq(Type Ty, XmmRegister dst, XmmRegister src) {
2044   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2045   emitUint8(0x66);
2046   emitRexRB(RexTypeIrrelevant, dst, src);
2047   emitUint8(0x0F);
2048   if (isByteSizedArithType(Ty)) {
2049     emitUint8(0x74);
2050   } else if (Ty == IceType_i16) {
2051     emitUint8(0x75);
2052   } else {
2053     emitUint8(0x76);
2054   }
2055   emitXmmRegisterOperand(dst, src);
2056 }
2057 
pcmpeq(Type Ty,XmmRegister dst,const AsmAddress & src)2058 void AssemblerX8664::pcmpeq(Type Ty, XmmRegister dst, const AsmAddress &src) {
2059   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2060   emitUint8(0x66);
2061   emitRex(RexTypeIrrelevant, src, dst);
2062   emitUint8(0x0F);
2063   if (isByteSizedArithType(Ty)) {
2064     emitUint8(0x74);
2065   } else if (Ty == IceType_i16) {
2066     emitUint8(0x75);
2067   } else {
2068     emitUint8(0x76);
2069   }
2070   emitOperand(gprEncoding(dst), src);
2071 }
2072 
pcmpgt(Type Ty,XmmRegister dst,XmmRegister src)2073 void AssemblerX8664::pcmpgt(Type Ty, XmmRegister dst, XmmRegister src) {
2074   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2075   emitUint8(0x66);
2076   emitRexRB(RexTypeIrrelevant, dst, src);
2077   emitUint8(0x0F);
2078   if (isByteSizedArithType(Ty)) {
2079     emitUint8(0x64);
2080   } else if (Ty == IceType_i16) {
2081     emitUint8(0x65);
2082   } else {
2083     emitUint8(0x66);
2084   }
2085   emitXmmRegisterOperand(dst, src);
2086 }
2087 
pcmpgt(Type Ty,XmmRegister dst,const AsmAddress & src)2088 void AssemblerX8664::pcmpgt(Type Ty, XmmRegister dst, const AsmAddress &src) {
2089   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2090   emitUint8(0x66);
2091   emitRex(RexTypeIrrelevant, src, dst);
2092   emitUint8(0x0F);
2093   if (isByteSizedArithType(Ty)) {
2094     emitUint8(0x64);
2095   } else if (Ty == IceType_i16) {
2096     emitUint8(0x65);
2097   } else {
2098     emitUint8(0x66);
2099   }
2100   emitOperand(gprEncoding(dst), src);
2101 }
2102 
round(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & mode)2103 void AssemblerX8664::round(Type Ty, XmmRegister dst, XmmRegister src,
2104                            const Immediate &mode) {
2105   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2106   emitUint8(0x66);
2107   emitRexRB(RexTypeIrrelevant, dst, src);
2108   emitUint8(0x0F);
2109   emitUint8(0x3A);
2110   switch (Ty) {
2111   case IceType_v4f32:
2112     emitUint8(0x08);
2113     break;
2114   case IceType_f32:
2115     emitUint8(0x0A);
2116     break;
2117   case IceType_f64:
2118     emitUint8(0x0B);
2119     break;
2120   default:
2121     assert(false && "Unsupported round operand type");
2122   }
2123   emitXmmRegisterOperand(dst, src);
2124   // Mask precision exeption.
2125   emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2126 }
2127 
round(Type Ty,XmmRegister dst,const AsmAddress & src,const Immediate & mode)2128 void AssemblerX8664::round(Type Ty, XmmRegister dst, const AsmAddress &src,
2129                            const Immediate &mode) {
2130   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2131   emitUint8(0x66);
2132   emitRex(RexTypeIrrelevant, src, dst);
2133   emitUint8(0x0F);
2134   emitUint8(0x3A);
2135   switch (Ty) {
2136   case IceType_v4f32:
2137     emitUint8(0x08);
2138     break;
2139   case IceType_f32:
2140     emitUint8(0x0A);
2141     break;
2142   case IceType_f64:
2143     emitUint8(0x0B);
2144     break;
2145   default:
2146     assert(false && "Unsupported round operand type");
2147   }
2148   emitOperand(gprEncoding(dst), src);
2149   // Mask precision exeption.
2150   emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2151 }
2152 
2153 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const Immediate & imm)2154 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg, const Immediate &imm) {
2155   static_assert(Tag < 8, "Tag must be between 0..7");
2156   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2157   if (Ty == IceType_i16)
2158     emitOperandSizeOverride();
2159   emitRexB(Ty, reg);
2160   if (isByteSizedType(Ty)) {
2161     emitComplexI8(Tag, AsmOperand(reg), imm);
2162   } else {
2163     emitComplex(Ty, Tag, AsmOperand(reg), imm);
2164   }
2165 }
2166 
2167 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg0,GPRRegister reg1)2168 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg0, GPRRegister reg1) {
2169   static_assert(Tag < 8, "Tag must be between 0..7");
2170   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2171   if (Ty == IceType_i16)
2172     emitOperandSizeOverride();
2173   emitRexRB(Ty, reg0, reg1);
2174   if (isByteSizedType(Ty))
2175     emitUint8(Tag * 8 + 2);
2176   else
2177     emitUint8(Tag * 8 + 3);
2178   emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
2179 }
2180 
2181 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const AsmAddress & address)2182 void AssemblerX8664::arith_int(Type Ty, GPRRegister reg,
2183                                const AsmAddress &address) {
2184   static_assert(Tag < 8, "Tag must be between 0..7");
2185   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2186   if (Ty == IceType_i16)
2187     emitOperandSizeOverride();
2188   emitRex(Ty, address, reg);
2189   if (isByteSizedType(Ty))
2190     emitUint8(Tag * 8 + 2);
2191   else
2192     emitUint8(Tag * 8 + 3);
2193   emitOperand(gprEncoding(reg), address);
2194 }
2195 
2196 template <uint32_t Tag>
arith_int(Type Ty,const AsmAddress & address,GPRRegister reg)2197 void AssemblerX8664::arith_int(Type Ty, const AsmAddress &address,
2198                                GPRRegister reg) {
2199   static_assert(Tag < 8, "Tag must be between 0..7");
2200   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2201   if (Ty == IceType_i16)
2202     emitOperandSizeOverride();
2203   emitRex(Ty, address, reg);
2204   if (isByteSizedType(Ty))
2205     emitUint8(Tag * 8 + 0);
2206   else
2207     emitUint8(Tag * 8 + 1);
2208   emitOperand(gprEncoding(reg), address);
2209 }
2210 
2211 template <uint32_t Tag>
arith_int(Type Ty,const AsmAddress & address,const Immediate & imm)2212 void AssemblerX8664::arith_int(Type Ty, const AsmAddress &address,
2213                                const Immediate &imm) {
2214   static_assert(Tag < 8, "Tag must be between 0..7");
2215   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2216   if (Ty == IceType_i16)
2217     emitOperandSizeOverride();
2218   emitRex(Ty, address, RexRegIrrelevant);
2219   if (isByteSizedType(Ty)) {
2220     emitComplexI8(Tag, address, imm);
2221   } else {
2222     emitComplex(Ty, Tag, address, imm);
2223   }
2224 }
2225 
cmp(Type Ty,GPRRegister reg,const Immediate & imm)2226 void AssemblerX8664::cmp(Type Ty, GPRRegister reg, const Immediate &imm) {
2227   arith_int<7>(Ty, reg, imm);
2228 }
2229 
cmp(Type Ty,GPRRegister reg0,GPRRegister reg1)2230 void AssemblerX8664::cmp(Type Ty, GPRRegister reg0, GPRRegister reg1) {
2231   arith_int<7>(Ty, reg0, reg1);
2232 }
2233 
cmp(Type Ty,GPRRegister reg,const AsmAddress & address)2234 void AssemblerX8664::cmp(Type Ty, GPRRegister reg, const AsmAddress &address) {
2235   arith_int<7>(Ty, reg, address);
2236 }
2237 
cmp(Type Ty,const AsmAddress & address,GPRRegister reg)2238 void AssemblerX8664::cmp(Type Ty, const AsmAddress &address, GPRRegister reg) {
2239   arith_int<7>(Ty, address, reg);
2240 }
2241 
cmp(Type Ty,const AsmAddress & address,const Immediate & imm)2242 void AssemblerX8664::cmp(Type Ty, const AsmAddress &address,
2243                          const Immediate &imm) {
2244   arith_int<7>(Ty, address, imm);
2245 }
2246 
test(Type Ty,GPRRegister reg1,GPRRegister reg2)2247 void AssemblerX8664::test(Type Ty, GPRRegister reg1, GPRRegister reg2) {
2248   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2249   if (Ty == IceType_i16)
2250     emitOperandSizeOverride();
2251   emitRexRB(Ty, reg1, reg2);
2252   if (isByteSizedType(Ty))
2253     emitUint8(0x84);
2254   else
2255     emitUint8(0x85);
2256   emitRegisterOperand(gprEncoding(reg1), gprEncoding(reg2));
2257 }
2258 
test(Type Ty,const AsmAddress & addr,GPRRegister reg)2259 void AssemblerX8664::test(Type Ty, const AsmAddress &addr, GPRRegister reg) {
2260   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2261   if (Ty == IceType_i16)
2262     emitOperandSizeOverride();
2263   emitRex(Ty, addr, reg);
2264   if (isByteSizedType(Ty))
2265     emitUint8(0x84);
2266   else
2267     emitUint8(0x85);
2268   emitOperand(gprEncoding(reg), addr);
2269 }
2270 
test(Type Ty,GPRRegister reg,const Immediate & immediate)2271 void AssemblerX8664::test(Type Ty, GPRRegister reg,
2272                           const Immediate &immediate) {
2273   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2274   // For registers that have a byte variant (EAX, EBX, ECX, and EDX) we only
2275   // test the byte register to keep the encoding short. This is legal even if
2276   // the register had high bits set since this only sets flags registers based
2277   // on the "AND" of the two operands, and the immediate had zeros at those
2278   // high bits.
2279   constexpr GPRRegister Last8BitGPR = GPRRegister::Encoded_Reg_r15d;
2280   if (immediate.is_uint8() && reg <= Last8BitGPR) {
2281     // Use zero-extended 8-bit immediate.
2282     emitRexB(Ty, reg);
2283     if (reg == RegX8664::Encoded_Reg_rax) {
2284       emitUint8(0xA8);
2285     } else {
2286       emitUint8(0xF6);
2287       emitUint8(0xC0 + gprEncoding(reg));
2288     }
2289     emitUint8(immediate.value() & 0xFF);
2290   } else if (reg == RegX8664::Encoded_Reg_rax) {
2291     // Use short form if the destination is EAX.
2292     if (Ty == IceType_i16)
2293       emitOperandSizeOverride();
2294     emitUint8(0xA9);
2295     emitImmediate(Ty, immediate);
2296   } else {
2297     if (Ty == IceType_i16)
2298       emitOperandSizeOverride();
2299     emitRexB(Ty, reg);
2300     emitUint8(0xF7);
2301     emitRegisterOperand(0, gprEncoding(reg));
2302     emitImmediate(Ty, immediate);
2303   }
2304 }
2305 
test(Type Ty,const AsmAddress & addr,const Immediate & immediate)2306 void AssemblerX8664::test(Type Ty, const AsmAddress &addr,
2307                           const Immediate &immediate) {
2308   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2309   // If the immediate is short, we only test the byte addr to keep the encoding
2310   // short.
2311   if (immediate.is_uint8()) {
2312     // Use zero-extended 8-bit immediate.
2313     emitRex(Ty, addr, RexRegIrrelevant);
2314     emitUint8(0xF6);
2315     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2316     emitOperand(0, addr, OffsetFromNextInstruction);
2317     emitUint8(immediate.value() & 0xFF);
2318   } else {
2319     if (Ty == IceType_i16)
2320       emitOperandSizeOverride();
2321     emitRex(Ty, addr, RexRegIrrelevant);
2322     emitUint8(0xF7);
2323     const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
2324     emitOperand(0, addr, OffsetFromNextInstruction);
2325     emitImmediate(Ty, immediate);
2326   }
2327 }
2328 
And(Type Ty,GPRRegister dst,GPRRegister src)2329 void AssemblerX8664::And(Type Ty, GPRRegister dst, GPRRegister src) {
2330   arith_int<4>(Ty, dst, src);
2331 }
2332 
And(Type Ty,GPRRegister dst,const AsmAddress & address)2333 void AssemblerX8664::And(Type Ty, GPRRegister dst, const AsmAddress &address) {
2334   arith_int<4>(Ty, dst, address);
2335 }
2336 
And(Type Ty,GPRRegister dst,const Immediate & imm)2337 void AssemblerX8664::And(Type Ty, GPRRegister dst, const Immediate &imm) {
2338   arith_int<4>(Ty, dst, imm);
2339 }
2340 
And(Type Ty,const AsmAddress & address,GPRRegister reg)2341 void AssemblerX8664::And(Type Ty, const AsmAddress &address, GPRRegister reg) {
2342   arith_int<4>(Ty, address, reg);
2343 }
2344 
And(Type Ty,const AsmAddress & address,const Immediate & imm)2345 void AssemblerX8664::And(Type Ty, const AsmAddress &address,
2346                          const Immediate &imm) {
2347   arith_int<4>(Ty, address, imm);
2348 }
2349 
Or(Type Ty,GPRRegister dst,GPRRegister src)2350 void AssemblerX8664::Or(Type Ty, GPRRegister dst, GPRRegister src) {
2351   arith_int<1>(Ty, dst, src);
2352 }
2353 
Or(Type Ty,GPRRegister dst,const AsmAddress & address)2354 void AssemblerX8664::Or(Type Ty, GPRRegister dst, const AsmAddress &address) {
2355   arith_int<1>(Ty, dst, address);
2356 }
2357 
Or(Type Ty,GPRRegister dst,const Immediate & imm)2358 void AssemblerX8664::Or(Type Ty, GPRRegister dst, const Immediate &imm) {
2359   arith_int<1>(Ty, dst, imm);
2360 }
2361 
Or(Type Ty,const AsmAddress & address,GPRRegister reg)2362 void AssemblerX8664::Or(Type Ty, const AsmAddress &address, GPRRegister reg) {
2363   arith_int<1>(Ty, address, reg);
2364 }
2365 
Or(Type Ty,const AsmAddress & address,const Immediate & imm)2366 void AssemblerX8664::Or(Type Ty, const AsmAddress &address,
2367                         const Immediate &imm) {
2368   arith_int<1>(Ty, address, imm);
2369 }
2370 
Xor(Type Ty,GPRRegister dst,GPRRegister src)2371 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, GPRRegister src) {
2372   arith_int<6>(Ty, dst, src);
2373 }
2374 
Xor(Type Ty,GPRRegister dst,const AsmAddress & address)2375 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, const AsmAddress &address) {
2376   arith_int<6>(Ty, dst, address);
2377 }
2378 
Xor(Type Ty,GPRRegister dst,const Immediate & imm)2379 void AssemblerX8664::Xor(Type Ty, GPRRegister dst, const Immediate &imm) {
2380   arith_int<6>(Ty, dst, imm);
2381 }
2382 
Xor(Type Ty,const AsmAddress & address,GPRRegister reg)2383 void AssemblerX8664::Xor(Type Ty, const AsmAddress &address, GPRRegister reg) {
2384   arith_int<6>(Ty, address, reg);
2385 }
2386 
Xor(Type Ty,const AsmAddress & address,const Immediate & imm)2387 void AssemblerX8664::Xor(Type Ty, const AsmAddress &address,
2388                          const Immediate &imm) {
2389   arith_int<6>(Ty, address, imm);
2390 }
2391 
add(Type Ty,GPRRegister dst,GPRRegister src)2392 void AssemblerX8664::add(Type Ty, GPRRegister dst, GPRRegister src) {
2393   arith_int<0>(Ty, dst, src);
2394 }
2395 
add(Type Ty,GPRRegister reg,const AsmAddress & address)2396 void AssemblerX8664::add(Type Ty, GPRRegister reg, const AsmAddress &address) {
2397   arith_int<0>(Ty, reg, address);
2398 }
2399 
add(Type Ty,GPRRegister reg,const Immediate & imm)2400 void AssemblerX8664::add(Type Ty, GPRRegister reg, const Immediate &imm) {
2401   arith_int<0>(Ty, reg, imm);
2402 }
2403 
add(Type Ty,const AsmAddress & address,GPRRegister reg)2404 void AssemblerX8664::add(Type Ty, const AsmAddress &address, GPRRegister reg) {
2405   arith_int<0>(Ty, address, reg);
2406 }
2407 
add(Type Ty,const AsmAddress & address,const Immediate & imm)2408 void AssemblerX8664::add(Type Ty, const AsmAddress &address,
2409                          const Immediate &imm) {
2410   arith_int<0>(Ty, address, imm);
2411 }
2412 
adc(Type Ty,GPRRegister dst,GPRRegister src)2413 void AssemblerX8664::adc(Type Ty, GPRRegister dst, GPRRegister src) {
2414   arith_int<2>(Ty, dst, src);
2415 }
2416 
adc(Type Ty,GPRRegister dst,const AsmAddress & address)2417 void AssemblerX8664::adc(Type Ty, GPRRegister dst, const AsmAddress &address) {
2418   arith_int<2>(Ty, dst, address);
2419 }
2420 
adc(Type Ty,GPRRegister reg,const Immediate & imm)2421 void AssemblerX8664::adc(Type Ty, GPRRegister reg, const Immediate &imm) {
2422   arith_int<2>(Ty, reg, imm);
2423 }
2424 
adc(Type Ty,const AsmAddress & address,GPRRegister reg)2425 void AssemblerX8664::adc(Type Ty, const AsmAddress &address, GPRRegister reg) {
2426   arith_int<2>(Ty, address, reg);
2427 }
2428 
adc(Type Ty,const AsmAddress & address,const Immediate & imm)2429 void AssemblerX8664::adc(Type Ty, const AsmAddress &address,
2430                          const Immediate &imm) {
2431   arith_int<2>(Ty, address, imm);
2432 }
2433 
sub(Type Ty,GPRRegister dst,GPRRegister src)2434 void AssemblerX8664::sub(Type Ty, GPRRegister dst, GPRRegister src) {
2435   arith_int<5>(Ty, dst, src);
2436 }
2437 
sub(Type Ty,GPRRegister reg,const AsmAddress & address)2438 void AssemblerX8664::sub(Type Ty, GPRRegister reg, const AsmAddress &address) {
2439   arith_int<5>(Ty, reg, address);
2440 }
2441 
sub(Type Ty,GPRRegister reg,const Immediate & imm)2442 void AssemblerX8664::sub(Type Ty, GPRRegister reg, const Immediate &imm) {
2443   arith_int<5>(Ty, reg, imm);
2444 }
2445 
sub(Type Ty,const AsmAddress & address,GPRRegister reg)2446 void AssemblerX8664::sub(Type Ty, const AsmAddress &address, GPRRegister reg) {
2447   arith_int<5>(Ty, address, reg);
2448 }
2449 
sub(Type Ty,const AsmAddress & address,const Immediate & imm)2450 void AssemblerX8664::sub(Type Ty, const AsmAddress &address,
2451                          const Immediate &imm) {
2452   arith_int<5>(Ty, address, imm);
2453 }
2454 
sbb(Type Ty,GPRRegister dst,GPRRegister src)2455 void AssemblerX8664::sbb(Type Ty, GPRRegister dst, GPRRegister src) {
2456   arith_int<3>(Ty, dst, src);
2457 }
2458 
sbb(Type Ty,GPRRegister dst,const AsmAddress & address)2459 void AssemblerX8664::sbb(Type Ty, GPRRegister dst, const AsmAddress &address) {
2460   arith_int<3>(Ty, dst, address);
2461 }
2462 
sbb(Type Ty,GPRRegister reg,const Immediate & imm)2463 void AssemblerX8664::sbb(Type Ty, GPRRegister reg, const Immediate &imm) {
2464   arith_int<3>(Ty, reg, imm);
2465 }
2466 
sbb(Type Ty,const AsmAddress & address,GPRRegister reg)2467 void AssemblerX8664::sbb(Type Ty, const AsmAddress &address, GPRRegister reg) {
2468   arith_int<3>(Ty, address, reg);
2469 }
2470 
sbb(Type Ty,const AsmAddress & address,const Immediate & imm)2471 void AssemblerX8664::sbb(Type Ty, const AsmAddress &address,
2472                          const Immediate &imm) {
2473   arith_int<3>(Ty, address, imm);
2474 }
2475 
cbw()2476 void AssemblerX8664::cbw() {
2477   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2478   emitOperandSizeOverride();
2479   emitUint8(0x98);
2480 }
2481 
cwd()2482 void AssemblerX8664::cwd() {
2483   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2484   emitOperandSizeOverride();
2485   emitUint8(0x99);
2486 }
2487 
cdq()2488 void AssemblerX8664::cdq() {
2489   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2490   emitUint8(0x99);
2491 }
2492 
cqo()2493 void AssemblerX8664::cqo() {
2494   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2495   emitRexB(RexTypeForceRexW, RexRegIrrelevant);
2496   emitUint8(0x99);
2497 }
2498 
div(Type Ty,GPRRegister reg)2499 void AssemblerX8664::div(Type Ty, GPRRegister reg) {
2500   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2501   if (Ty == IceType_i16)
2502     emitOperandSizeOverride();
2503   emitRexB(Ty, reg);
2504   if (isByteSizedArithType(Ty))
2505     emitUint8(0xF6);
2506   else
2507     emitUint8(0xF7);
2508   emitRegisterOperand(6, gprEncoding(reg));
2509 }
2510 
div(Type Ty,const AsmAddress & addr)2511 void AssemblerX8664::div(Type Ty, const AsmAddress &addr) {
2512   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2513   if (Ty == IceType_i16)
2514     emitOperandSizeOverride();
2515   emitRex(Ty, addr, RexRegIrrelevant);
2516   if (isByteSizedArithType(Ty))
2517     emitUint8(0xF6);
2518   else
2519     emitUint8(0xF7);
2520   emitOperand(6, addr);
2521 }
2522 
idiv(Type Ty,GPRRegister reg)2523 void AssemblerX8664::idiv(Type Ty, GPRRegister reg) {
2524   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2525   if (Ty == IceType_i16)
2526     emitOperandSizeOverride();
2527   emitRexB(Ty, reg);
2528   if (isByteSizedArithType(Ty))
2529     emitUint8(0xF6);
2530   else
2531     emitUint8(0xF7);
2532   emitRegisterOperand(7, gprEncoding(reg));
2533 }
2534 
idiv(Type Ty,const AsmAddress & addr)2535 void AssemblerX8664::idiv(Type Ty, const AsmAddress &addr) {
2536   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2537   if (Ty == IceType_i16)
2538     emitOperandSizeOverride();
2539   emitRex(Ty, addr, RexRegIrrelevant);
2540   if (isByteSizedArithType(Ty))
2541     emitUint8(0xF6);
2542   else
2543     emitUint8(0xF7);
2544   emitOperand(7, addr);
2545 }
2546 
imul(Type Ty,GPRRegister dst,GPRRegister src)2547 void AssemblerX8664::imul(Type Ty, GPRRegister dst, GPRRegister src) {
2548   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2549   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2550   if (Ty == IceType_i16)
2551     emitOperandSizeOverride();
2552   emitRexRB(Ty, dst, src);
2553   emitUint8(0x0F);
2554   emitUint8(0xAF);
2555   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2556 }
2557 
imul(Type Ty,GPRRegister reg,const AsmAddress & address)2558 void AssemblerX8664::imul(Type Ty, GPRRegister reg, const AsmAddress &address) {
2559   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2560   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2561   if (Ty == IceType_i16)
2562     emitOperandSizeOverride();
2563   emitRex(Ty, address, reg);
2564   emitUint8(0x0F);
2565   emitUint8(0xAF);
2566   emitOperand(gprEncoding(reg), address);
2567 }
2568 
imul(Type Ty,GPRRegister reg,const Immediate & imm)2569 void AssemblerX8664::imul(Type Ty, GPRRegister reg, const Immediate &imm) {
2570   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2571   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2572   if (Ty == IceType_i16)
2573     emitOperandSizeOverride();
2574   emitRexRB(Ty, reg, reg);
2575   if (imm.is_int8()) {
2576     emitUint8(0x6B);
2577     emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
2578     emitUint8(imm.value() & 0xFF);
2579   } else {
2580     emitUint8(0x69);
2581     emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
2582     emitImmediate(Ty, imm);
2583   }
2584 }
2585 
imul(Type Ty,GPRRegister reg)2586 void AssemblerX8664::imul(Type Ty, GPRRegister reg) {
2587   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2588   if (Ty == IceType_i16)
2589     emitOperandSizeOverride();
2590   emitRexB(Ty, reg);
2591   if (isByteSizedArithType(Ty))
2592     emitUint8(0xF6);
2593   else
2594     emitUint8(0xF7);
2595   emitRegisterOperand(5, gprEncoding(reg));
2596 }
2597 
imul(Type Ty,const AsmAddress & address)2598 void AssemblerX8664::imul(Type Ty, const AsmAddress &address) {
2599   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2600   if (Ty == IceType_i16)
2601     emitOperandSizeOverride();
2602   emitRex(Ty, address, RexRegIrrelevant);
2603   if (isByteSizedArithType(Ty))
2604     emitUint8(0xF6);
2605   else
2606     emitUint8(0xF7);
2607   emitOperand(5, address);
2608 }
2609 
imul(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2610 void AssemblerX8664::imul(Type Ty, GPRRegister dst, GPRRegister src,
2611                           const Immediate &imm) {
2612   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2613   assert(Ty == IceType_i16 || Ty == IceType_i32);
2614   if (Ty == IceType_i16)
2615     emitOperandSizeOverride();
2616   emitRexRB(Ty, dst, src);
2617   if (imm.is_int8()) {
2618     emitUint8(0x6B);
2619     emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2620     emitUint8(imm.value() & 0xFF);
2621   } else {
2622     emitUint8(0x69);
2623     emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2624     emitImmediate(Ty, imm);
2625   }
2626 }
2627 
imul(Type Ty,GPRRegister dst,const AsmAddress & address,const Immediate & imm)2628 void AssemblerX8664::imul(Type Ty, GPRRegister dst, const AsmAddress &address,
2629                           const Immediate &imm) {
2630   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2631   assert(Ty == IceType_i16 || Ty == IceType_i32);
2632   if (Ty == IceType_i16)
2633     emitOperandSizeOverride();
2634   emitRex(Ty, address, dst);
2635   if (imm.is_int8()) {
2636     emitUint8(0x6B);
2637     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2638     emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
2639     emitUint8(imm.value() & 0xFF);
2640   } else {
2641     emitUint8(0x69);
2642     const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
2643     emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
2644     emitImmediate(Ty, imm);
2645   }
2646 }
2647 
mul(Type Ty,GPRRegister reg)2648 void AssemblerX8664::mul(Type Ty, GPRRegister reg) {
2649   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2650   if (Ty == IceType_i16)
2651     emitOperandSizeOverride();
2652   emitRexB(Ty, reg);
2653   if (isByteSizedArithType(Ty))
2654     emitUint8(0xF6);
2655   else
2656     emitUint8(0xF7);
2657   emitRegisterOperand(4, gprEncoding(reg));
2658 }
2659 
mul(Type Ty,const AsmAddress & address)2660 void AssemblerX8664::mul(Type Ty, const AsmAddress &address) {
2661   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2662   if (Ty == IceType_i16)
2663     emitOperandSizeOverride();
2664   emitRex(Ty, address, RexRegIrrelevant);
2665   if (isByteSizedArithType(Ty))
2666     emitUint8(0xF6);
2667   else
2668     emitUint8(0xF7);
2669   emitOperand(4, address);
2670 }
2671 
incl(const AsmAddress & address)2672 void AssemblerX8664::incl(const AsmAddress &address) {
2673   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2674   emitRex(IceType_i32, address, RexRegIrrelevant);
2675   emitUint8(0xFF);
2676   emitOperand(0, address);
2677 }
2678 
decl(const AsmAddress & address)2679 void AssemblerX8664::decl(const AsmAddress &address) {
2680   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2681   emitRex(IceType_i32, address, RexRegIrrelevant);
2682   emitUint8(0xFF);
2683   emitOperand(1, address);
2684 }
2685 
rol(Type Ty,GPRRegister reg,const Immediate & imm)2686 void AssemblerX8664::rol(Type Ty, GPRRegister reg, const Immediate &imm) {
2687   emitGenericShift(0, Ty, reg, imm);
2688 }
2689 
rol(Type Ty,GPRRegister operand,GPRRegister shifter)2690 void AssemblerX8664::rol(Type Ty, GPRRegister operand, GPRRegister shifter) {
2691   emitGenericShift(0, Ty, AsmOperand(operand), shifter);
2692 }
2693 
rol(Type Ty,const AsmAddress & operand,GPRRegister shifter)2694 void AssemblerX8664::rol(Type Ty, const AsmAddress &operand,
2695                          GPRRegister shifter) {
2696   emitGenericShift(0, Ty, operand, shifter);
2697 }
2698 
shl(Type Ty,GPRRegister reg,const Immediate & imm)2699 void AssemblerX8664::shl(Type Ty, GPRRegister reg, const Immediate &imm) {
2700   emitGenericShift(4, Ty, reg, imm);
2701 }
2702 
shl(Type Ty,GPRRegister operand,GPRRegister shifter)2703 void AssemblerX8664::shl(Type Ty, GPRRegister operand, GPRRegister shifter) {
2704   emitGenericShift(4, Ty, AsmOperand(operand), shifter);
2705 }
2706 
shl(Type Ty,const AsmAddress & operand,GPRRegister shifter)2707 void AssemblerX8664::shl(Type Ty, const AsmAddress &operand,
2708                          GPRRegister shifter) {
2709   emitGenericShift(4, Ty, operand, shifter);
2710 }
2711 
shr(Type Ty,GPRRegister reg,const Immediate & imm)2712 void AssemblerX8664::shr(Type Ty, GPRRegister reg, const Immediate &imm) {
2713   emitGenericShift(5, Ty, reg, imm);
2714 }
2715 
shr(Type Ty,GPRRegister operand,GPRRegister shifter)2716 void AssemblerX8664::shr(Type Ty, GPRRegister operand, GPRRegister shifter) {
2717   emitGenericShift(5, Ty, AsmOperand(operand), shifter);
2718 }
2719 
shr(Type Ty,const AsmAddress & operand,GPRRegister shifter)2720 void AssemblerX8664::shr(Type Ty, const AsmAddress &operand,
2721                          GPRRegister shifter) {
2722   emitGenericShift(5, Ty, operand, shifter);
2723 }
2724 
sar(Type Ty,GPRRegister reg,const Immediate & imm)2725 void AssemblerX8664::sar(Type Ty, GPRRegister reg, const Immediate &imm) {
2726   emitGenericShift(7, Ty, reg, imm);
2727 }
2728 
sar(Type Ty,GPRRegister operand,GPRRegister shifter)2729 void AssemblerX8664::sar(Type Ty, GPRRegister operand, GPRRegister shifter) {
2730   emitGenericShift(7, Ty, AsmOperand(operand), shifter);
2731 }
2732 
sar(Type Ty,const AsmAddress & address,GPRRegister shifter)2733 void AssemblerX8664::sar(Type Ty, const AsmAddress &address,
2734                          GPRRegister shifter) {
2735   emitGenericShift(7, Ty, address, shifter);
2736 }
2737 
shld(Type Ty,GPRRegister dst,GPRRegister src)2738 void AssemblerX8664::shld(Type Ty, GPRRegister dst, GPRRegister src) {
2739   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2740   assert(Ty == IceType_i16 || Ty == IceType_i32);
2741   if (Ty == IceType_i16)
2742     emitOperandSizeOverride();
2743   emitRexRB(Ty, src, dst);
2744   emitUint8(0x0F);
2745   emitUint8(0xA5);
2746   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2747 }
2748 
shld(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2749 void AssemblerX8664::shld(Type Ty, GPRRegister dst, GPRRegister src,
2750                           const Immediate &imm) {
2751   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2752   assert(Ty == IceType_i16 || Ty == IceType_i32);
2753   assert(imm.is_int8());
2754   if (Ty == IceType_i16)
2755     emitOperandSizeOverride();
2756   emitRexRB(Ty, src, dst);
2757   emitUint8(0x0F);
2758   emitUint8(0xA4);
2759   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2760   emitUint8(imm.value() & 0xFF);
2761 }
2762 
shld(Type Ty,const AsmAddress & operand,GPRRegister src)2763 void AssemblerX8664::shld(Type Ty, const AsmAddress &operand, GPRRegister src) {
2764   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2765   assert(Ty == IceType_i16 || Ty == IceType_i32);
2766   if (Ty == IceType_i16)
2767     emitOperandSizeOverride();
2768   emitRex(Ty, operand, src);
2769   emitUint8(0x0F);
2770   emitUint8(0xA5);
2771   emitOperand(gprEncoding(src), operand);
2772 }
2773 
shrd(Type Ty,GPRRegister dst,GPRRegister src)2774 void AssemblerX8664::shrd(Type Ty, GPRRegister dst, GPRRegister src) {
2775   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2776   assert(Ty == IceType_i16 || Ty == IceType_i32);
2777   if (Ty == IceType_i16)
2778     emitOperandSizeOverride();
2779   emitRexRB(Ty, src, dst);
2780   emitUint8(0x0F);
2781   emitUint8(0xAD);
2782   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2783 }
2784 
shrd(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)2785 void AssemblerX8664::shrd(Type Ty, GPRRegister dst, GPRRegister src,
2786                           const Immediate &imm) {
2787   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2788   assert(Ty == IceType_i16 || Ty == IceType_i32);
2789   assert(imm.is_int8());
2790   if (Ty == IceType_i16)
2791     emitOperandSizeOverride();
2792   emitRexRB(Ty, src, dst);
2793   emitUint8(0x0F);
2794   emitUint8(0xAC);
2795   emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
2796   emitUint8(imm.value() & 0xFF);
2797 }
2798 
shrd(Type Ty,const AsmAddress & dst,GPRRegister src)2799 void AssemblerX8664::shrd(Type Ty, const AsmAddress &dst, GPRRegister src) {
2800   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2801   assert(Ty == IceType_i16 || Ty == IceType_i32);
2802   if (Ty == IceType_i16)
2803     emitOperandSizeOverride();
2804   emitRex(Ty, dst, src);
2805   emitUint8(0x0F);
2806   emitUint8(0xAD);
2807   emitOperand(gprEncoding(src), dst);
2808 }
2809 
neg(Type Ty,GPRRegister reg)2810 void AssemblerX8664::neg(Type Ty, GPRRegister reg) {
2811   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2812   if (Ty == IceType_i16)
2813     emitOperandSizeOverride();
2814   emitRexB(Ty, reg);
2815   if (isByteSizedArithType(Ty))
2816     emitUint8(0xF6);
2817   else
2818     emitUint8(0xF7);
2819   emitRegisterOperand(3, gprEncoding(reg));
2820 }
2821 
neg(Type Ty,const AsmAddress & addr)2822 void AssemblerX8664::neg(Type Ty, const AsmAddress &addr) {
2823   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2824   if (Ty == IceType_i16)
2825     emitOperandSizeOverride();
2826   emitRex(Ty, addr, RexRegIrrelevant);
2827   if (isByteSizedArithType(Ty))
2828     emitUint8(0xF6);
2829   else
2830     emitUint8(0xF7);
2831   emitOperand(3, addr);
2832 }
2833 
notl(GPRRegister reg)2834 void AssemblerX8664::notl(GPRRegister reg) {
2835   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2836   emitRexB(IceType_i32, reg);
2837   emitUint8(0xF7);
2838   emitUint8(0xD0 | gprEncoding(reg));
2839 }
2840 
bswap(Type Ty,GPRRegister reg)2841 void AssemblerX8664::bswap(Type Ty, GPRRegister reg) {
2842   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2843   assert(Ty == IceType_i32 || Ty == IceType_i64);
2844   emitRexB(Ty, reg);
2845   emitUint8(0x0F);
2846   emitUint8(0xC8 | gprEncoding(reg));
2847 }
2848 
bsf(Type Ty,GPRRegister dst,GPRRegister src)2849 void AssemblerX8664::bsf(Type Ty, GPRRegister dst, GPRRegister src) {
2850   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2851   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2852   if (Ty == IceType_i16)
2853     emitOperandSizeOverride();
2854   emitRexRB(Ty, dst, src);
2855   emitUint8(0x0F);
2856   emitUint8(0xBC);
2857   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2858 }
2859 
bsf(Type Ty,GPRRegister dst,const AsmAddress & src)2860 void AssemblerX8664::bsf(Type Ty, GPRRegister dst, const AsmAddress &src) {
2861   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2862   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2863   if (Ty == IceType_i16)
2864     emitOperandSizeOverride();
2865   emitRex(Ty, src, dst);
2866   emitUint8(0x0F);
2867   emitUint8(0xBC);
2868   emitOperand(gprEncoding(dst), src);
2869 }
2870 
bsr(Type Ty,GPRRegister dst,GPRRegister src)2871 void AssemblerX8664::bsr(Type Ty, GPRRegister dst, GPRRegister src) {
2872   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2873   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2874   if (Ty == IceType_i16)
2875     emitOperandSizeOverride();
2876   emitRexRB(Ty, dst, src);
2877   emitUint8(0x0F);
2878   emitUint8(0xBD);
2879   emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
2880 }
2881 
bsr(Type Ty,GPRRegister dst,const AsmAddress & src)2882 void AssemblerX8664::bsr(Type Ty, GPRRegister dst, const AsmAddress &src) {
2883   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2884   assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
2885   if (Ty == IceType_i16)
2886     emitOperandSizeOverride();
2887   emitRex(Ty, src, dst);
2888   emitUint8(0x0F);
2889   emitUint8(0xBD);
2890   emitOperand(gprEncoding(dst), src);
2891 }
2892 
bt(GPRRegister base,GPRRegister offset)2893 void AssemblerX8664::bt(GPRRegister base, GPRRegister offset) {
2894   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2895   emitRexRB(IceType_i32, offset, base);
2896   emitUint8(0x0F);
2897   emitUint8(0xA3);
2898   emitRegisterOperand(gprEncoding(offset), gprEncoding(base));
2899 }
2900 
ret()2901 void AssemblerX8664::ret() {
2902   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2903   emitUint8(0xC3);
2904 }
2905 
ret(const Immediate & imm)2906 void AssemblerX8664::ret(const Immediate &imm) {
2907   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2908   emitUint8(0xC2);
2909   assert(imm.is_uint16());
2910   emitUint8(imm.value() & 0xFF);
2911   emitUint8((imm.value() >> 8) & 0xFF);
2912 }
2913 
nop(int size)2914 void AssemblerX8664::nop(int size) {
2915   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2916   // There are nops up to size 15, but for now just provide up to size 8.
2917   assert(0 < size && size <= MAX_NOP_SIZE);
2918   switch (size) {
2919   case 1:
2920     emitUint8(0x90);
2921     break;
2922   case 2:
2923     emitUint8(0x66);
2924     emitUint8(0x90);
2925     break;
2926   case 3:
2927     emitUint8(0x0F);
2928     emitUint8(0x1F);
2929     emitUint8(0x00);
2930     break;
2931   case 4:
2932     emitUint8(0x0F);
2933     emitUint8(0x1F);
2934     emitUint8(0x40);
2935     emitUint8(0x00);
2936     break;
2937   case 5:
2938     emitUint8(0x0F);
2939     emitUint8(0x1F);
2940     emitUint8(0x44);
2941     emitUint8(0x00);
2942     emitUint8(0x00);
2943     break;
2944   case 6:
2945     emitUint8(0x66);
2946     emitUint8(0x0F);
2947     emitUint8(0x1F);
2948     emitUint8(0x44);
2949     emitUint8(0x00);
2950     emitUint8(0x00);
2951     break;
2952   case 7:
2953     emitUint8(0x0F);
2954     emitUint8(0x1F);
2955     emitUint8(0x80);
2956     emitUint8(0x00);
2957     emitUint8(0x00);
2958     emitUint8(0x00);
2959     emitUint8(0x00);
2960     break;
2961   case 8:
2962     emitUint8(0x0F);
2963     emitUint8(0x1F);
2964     emitUint8(0x84);
2965     emitUint8(0x00);
2966     emitUint8(0x00);
2967     emitUint8(0x00);
2968     emitUint8(0x00);
2969     emitUint8(0x00);
2970     break;
2971   default:
2972     llvm_unreachable("Unimplemented");
2973   }
2974 }
2975 
int3()2976 void AssemblerX8664::int3() {
2977   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2978   emitUint8(0xCC);
2979 }
2980 
hlt()2981 void AssemblerX8664::hlt() {
2982   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2983   emitUint8(0xF4);
2984 }
2985 
ud2()2986 void AssemblerX8664::ud2() {
2987   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2988   emitUint8(0x0F);
2989   emitUint8(0x0B);
2990 }
2991 
j(BrCond condition,Label * label,bool near)2992 void AssemblerX8664::j(BrCond condition, Label *label, bool near) {
2993   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2994   if (label->isBound()) {
2995     static const int kShortSize = 2;
2996     static const int kLongSize = 6;
2997     intptr_t offset = label->getPosition() - Buffer.size();
2998     assert(offset <= 0);
2999     if (Utils::IsInt(8, offset - kShortSize)) {
3000       emitUint8(0x70 + condition);
3001       emitUint8((offset - kShortSize) & 0xFF);
3002     } else {
3003       emitUint8(0x0F);
3004       emitUint8(0x80 + condition);
3005       emitInt32(offset - kLongSize);
3006     }
3007   } else if (near) {
3008     emitUint8(0x70 + condition);
3009     emitNearLabelLink(label);
3010   } else {
3011     emitUint8(0x0F);
3012     emitUint8(0x80 + condition);
3013     emitLabelLink(label);
3014   }
3015 }
3016 
j(BrCond condition,const ConstantRelocatable * label)3017 void AssemblerX8664::j(BrCond condition, const ConstantRelocatable *label) {
3018   llvm::report_fatal_error("Untested - please verify and then reenable.");
3019   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3020   emitUint8(0x0F);
3021   emitUint8(0x80 + condition);
3022   auto *Fixup = this->createFixup(FK_PcRel, label);
3023   Fixup->set_addend(-4);
3024   emitFixup(Fixup);
3025   emitInt32(0);
3026 }
3027 
jmp(GPRRegister reg)3028 void AssemblerX8664::jmp(GPRRegister reg) {
3029   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3030   emitRexB(RexTypeIrrelevant, reg);
3031   emitUint8(0xFF);
3032   emitRegisterOperand(4, gprEncoding(reg));
3033 }
3034 
jmp(Label * label,bool near)3035 void AssemblerX8664::jmp(Label *label, bool near) {
3036   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3037   if (label->isBound()) {
3038     static const int kShortSize = 2;
3039     static const int kLongSize = 5;
3040     intptr_t offset = label->getPosition() - Buffer.size();
3041     assert(offset <= 0);
3042     if (Utils::IsInt(8, offset - kShortSize)) {
3043       emitUint8(0xEB);
3044       emitUint8((offset - kShortSize) & 0xFF);
3045     } else {
3046       emitUint8(0xE9);
3047       emitInt32(offset - kLongSize);
3048     }
3049   } else if (near) {
3050     emitUint8(0xEB);
3051     emitNearLabelLink(label);
3052   } else {
3053     emitUint8(0xE9);
3054     emitLabelLink(label);
3055   }
3056 }
3057 
jmp(const ConstantRelocatable * label)3058 void AssemblerX8664::jmp(const ConstantRelocatable *label) {
3059   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3060   emitUint8(0xE9);
3061   auto *Fixup = this->createFixup(FK_PcRel, label);
3062   Fixup->set_addend(-4);
3063   emitFixup(Fixup);
3064   emitInt32(0);
3065 }
3066 
jmp(const Immediate & abs_address)3067 void AssemblerX8664::jmp(const Immediate &abs_address) {
3068   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3069   emitUint8(0xE9);
3070   AssemblerFixup *Fixup = createFixup(FK_PcRel, AssemblerFixup::NullSymbol);
3071   Fixup->set_addend(abs_address.value() - 4);
3072   emitFixup(Fixup);
3073   emitInt32(0);
3074 }
3075 
mfence()3076 void AssemblerX8664::mfence() {
3077   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3078   emitUint8(0x0F);
3079   emitUint8(0xAE);
3080   emitUint8(0xF0);
3081 }
3082 
lock()3083 void AssemblerX8664::lock() {
3084   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3085   emitUint8(0xF0);
3086 }
3087 
cmpxchg(Type Ty,const AsmAddress & address,GPRRegister reg,bool Locked)3088 void AssemblerX8664::cmpxchg(Type Ty, const AsmAddress &address,
3089                              GPRRegister reg, bool Locked) {
3090   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3091   if (Ty == IceType_i16)
3092     emitOperandSizeOverride();
3093   if (Locked)
3094     emitUint8(0xF0);
3095   emitRex(Ty, address, reg);
3096   emitUint8(0x0F);
3097   if (isByteSizedArithType(Ty))
3098     emitUint8(0xB0);
3099   else
3100     emitUint8(0xB1);
3101   emitOperand(gprEncoding(reg), address);
3102 }
3103 
cmpxchg8b(const AsmAddress & address,bool Locked)3104 void AssemblerX8664::cmpxchg8b(const AsmAddress &address, bool Locked) {
3105   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3106   if (Locked)
3107     emitUint8(0xF0);
3108   emitRex(IceType_i32, address, RexRegIrrelevant);
3109   emitUint8(0x0F);
3110   emitUint8(0xC7);
3111   emitOperand(1, address);
3112 }
3113 
xadd(Type Ty,const AsmAddress & addr,GPRRegister reg,bool Locked)3114 void AssemblerX8664::xadd(Type Ty, const AsmAddress &addr, GPRRegister reg,
3115                           bool Locked) {
3116   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3117   if (Ty == IceType_i16)
3118     emitOperandSizeOverride();
3119   if (Locked)
3120     emitUint8(0xF0);
3121   emitRex(Ty, addr, reg);
3122   emitUint8(0x0F);
3123   if (isByteSizedArithType(Ty))
3124     emitUint8(0xC0);
3125   else
3126     emitUint8(0xC1);
3127   emitOperand(gprEncoding(reg), addr);
3128 }
3129 
xchg(Type Ty,GPRRegister reg0,GPRRegister reg1)3130 void AssemblerX8664::xchg(Type Ty, GPRRegister reg0, GPRRegister reg1) {
3131   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3132   if (Ty == IceType_i16)
3133     emitOperandSizeOverride();
3134   // Use short form if either register is RAX.
3135   if (reg0 == RegX8664::Encoded_Reg_rax) {
3136     emitRexB(Ty, reg1);
3137     emitUint8(0x90 + gprEncoding(reg1));
3138   } else if (reg1 == RegX8664::Encoded_Reg_rax) {
3139     emitRexB(Ty, reg0);
3140     emitUint8(0x90 + gprEncoding(reg0));
3141   } else {
3142     emitRexRB(Ty, reg0, reg1);
3143     if (isByteSizedArithType(Ty))
3144       emitUint8(0x86);
3145     else
3146       emitUint8(0x87);
3147     emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
3148   }
3149 }
3150 
xchg(Type Ty,const AsmAddress & addr,GPRRegister reg)3151 void AssemblerX8664::xchg(Type Ty, const AsmAddress &addr, GPRRegister reg) {
3152   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3153   if (Ty == IceType_i16)
3154     emitOperandSizeOverride();
3155   emitRex(Ty, addr, reg);
3156   if (isByteSizedArithType(Ty))
3157     emitUint8(0x86);
3158   else
3159     emitUint8(0x87);
3160   emitOperand(gprEncoding(reg), addr);
3161 }
3162 
iaca_start()3163 void AssemblerX8664::iaca_start() {
3164   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3165   emitUint8(0x0F);
3166   emitUint8(0x0B);
3167 
3168   // mov $111, ebx
3169   constexpr GPRRegister dst = GPRRegister::Encoded_Reg_ebx;
3170   constexpr Type Ty = IceType_i32;
3171   emitRexB(Ty, dst);
3172   emitUint8(0xB8 + gprEncoding(dst));
3173   emitImmediate(Ty, Immediate(111));
3174 
3175   emitUint8(0x64);
3176   emitUint8(0x67);
3177   emitUint8(0x90);
3178 }
3179 
iaca_end()3180 void AssemblerX8664::iaca_end() {
3181   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3182 
3183   // mov $222, ebx
3184   constexpr GPRRegister dst = GPRRegister::Encoded_Reg_ebx;
3185   constexpr Type Ty = IceType_i32;
3186   emitRexB(Ty, dst);
3187   emitUint8(0xB8 + gprEncoding(dst));
3188   emitImmediate(Ty, Immediate(222));
3189 
3190   emitUint8(0x64);
3191   emitUint8(0x67);
3192   emitUint8(0x90);
3193 
3194   emitUint8(0x0F);
3195   emitUint8(0x0B);
3196 }
3197 
emitSegmentOverride(uint8_t prefix)3198 void AssemblerX8664::emitSegmentOverride(uint8_t prefix) {
3199   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3200   emitUint8(prefix);
3201 }
3202 
align(intptr_t alignment,intptr_t offset)3203 void AssemblerX8664::align(intptr_t alignment, intptr_t offset) {
3204   assert(llvm::isPowerOf2_32(alignment));
3205   intptr_t pos = offset + Buffer.getPosition();
3206   intptr_t mod = pos & (alignment - 1);
3207   if (mod == 0) {
3208     return;
3209   }
3210   intptr_t bytes_needed = alignment - mod;
3211   while (bytes_needed > MAX_NOP_SIZE) {
3212     nop(MAX_NOP_SIZE);
3213     bytes_needed -= MAX_NOP_SIZE;
3214   }
3215   if (bytes_needed) {
3216     nop(bytes_needed);
3217   }
3218   assert(((offset + Buffer.getPosition()) & (alignment - 1)) == 0);
3219 }
3220 
bind(Label * L)3221 void AssemblerX8664::bind(Label *L) {
3222   const intptr_t Bound = Buffer.size();
3223   assert(!L->isBound()); // Labels can only be bound once.
3224   while (L->isLinked()) {
3225     const intptr_t Position = L->getLinkPosition();
3226     const intptr_t Next = Buffer.load<int32_t>(Position);
3227     const intptr_t Offset = Bound - (Position + 4);
3228     Buffer.store<int32_t>(Position, Offset);
3229     L->Position = Next;
3230   }
3231   while (L->hasNear()) {
3232     intptr_t Position = L->getNearPosition();
3233     const intptr_t Offset = Bound - (Position + 1);
3234     assert(Utils::IsInt(8, Offset));
3235     Buffer.store<int8_t>(Position, Offset);
3236   }
3237   L->bindTo(Bound);
3238 }
3239 
emitOperand(int rm,const AsmOperand & operand,RelocOffsetT Addend)3240 void AssemblerX8664::emitOperand(int rm, const AsmOperand &operand,
3241                                  RelocOffsetT Addend) {
3242   assert(rm >= 0 && rm < 8);
3243   const intptr_t length = operand.length_;
3244   assert(length > 0);
3245   intptr_t displacement_start = 1;
3246   // Emit the ModRM byte updated with the given RM value.
3247   assert((operand.encoding_[0] & 0x38) == 0);
3248   emitUint8(operand.encoding_[0] + (rm << 3));
3249   // Whenever the addressing mode is not register indirect, using esp == 0x4
3250   // as the register operation indicates an SIB byte follows.
3251   if (((operand.encoding_[0] & 0xc0) != 0xc0) &&
3252       ((operand.encoding_[0] & 0x07) == 0x04)) {
3253     emitUint8(operand.encoding_[1]);
3254     displacement_start = 2;
3255   }
3256 
3257   AssemblerFixup *Fixup = operand.fixup();
3258   if (Fixup == nullptr) {
3259     for (intptr_t i = displacement_start; i < length; i++) {
3260       emitUint8(operand.encoding_[i]);
3261     }
3262     return;
3263   }
3264 
3265   // Emit the fixup, and a dummy 4-byte immediate. Note that the Disp32 in
3266   // operand.encoding_[i, i+1, i+2, i+3] is part of the constant relocatable
3267   // used to create the fixup, so there's no need to add it to the addend.
3268   assert(length - displacement_start == 4);
3269   if (fixupIsPCRel(Fixup->kind())) {
3270     Fixup->set_addend(Fixup->get_addend() - Addend);
3271   } else {
3272     Fixup->set_addend(Fixup->get_addend());
3273   }
3274   emitFixup(Fixup);
3275   emitInt32(0);
3276 }
3277 
emitImmediate(Type Ty,const Immediate & imm)3278 void AssemblerX8664::emitImmediate(Type Ty, const Immediate &imm) {
3279   auto *const Fixup = imm.fixup();
3280   if (Ty == IceType_i16) {
3281     assert(Fixup == nullptr);
3282     emitInt16(imm.value());
3283     return;
3284   }
3285 
3286   if (Fixup == nullptr) {
3287     emitInt32(imm.value());
3288     return;
3289   }
3290 
3291   Fixup->set_addend(Fixup->get_addend() + imm.value());
3292   emitFixup(Fixup);
3293   emitInt32(0);
3294 }
3295 
emitComplexI8(int rm,const AsmOperand & operand,const Immediate & immediate)3296 void AssemblerX8664::emitComplexI8(int rm, const AsmOperand &operand,
3297                                    const Immediate &immediate) {
3298   assert(rm >= 0 && rm < 8);
3299   assert(immediate.is_int8());
3300   if (operand.IsRegister(RegX8664::Encoded_Reg_rax)) {
3301     // Use short form if the destination is al.
3302     emitUint8(0x04 + (rm << 3));
3303     emitUint8(immediate.value() & 0xFF);
3304   } else {
3305     // Use sign-extended 8-bit immediate.
3306     emitUint8(0x80);
3307     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3308     emitOperand(rm, operand, OffsetFromNextInstruction);
3309     emitUint8(immediate.value() & 0xFF);
3310   }
3311 }
3312 
emitComplex(Type Ty,int rm,const AsmOperand & operand,const Immediate & immediate)3313 void AssemblerX8664::emitComplex(Type Ty, int rm, const AsmOperand &operand,
3314                                  const Immediate &immediate) {
3315   assert(rm >= 0 && rm < 8);
3316   if (immediate.is_int8()) {
3317     // Use sign-extended 8-bit immediate.
3318     emitUint8(0x83);
3319     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3320     emitOperand(rm, operand, OffsetFromNextInstruction);
3321     emitUint8(immediate.value() & 0xFF);
3322   } else if (operand.IsRegister(RegX8664::Encoded_Reg_rax)) {
3323     // Use short form if the destination is eax.
3324     emitUint8(0x05 + (rm << 3));
3325     emitImmediate(Ty, immediate);
3326   } else {
3327     emitUint8(0x81);
3328     const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
3329     emitOperand(rm, operand, OffsetFromNextInstruction);
3330     emitImmediate(Ty, immediate);
3331   }
3332 }
3333 
emitLabel(Label * label,intptr_t instruction_size)3334 void AssemblerX8664::emitLabel(Label *label, intptr_t instruction_size) {
3335   if (label->isBound()) {
3336     intptr_t offset = label->getPosition() - Buffer.size();
3337     assert(offset <= 0);
3338     emitInt32(offset - instruction_size);
3339   } else {
3340     emitLabelLink(label);
3341   }
3342 }
3343 
emitLabelLink(Label * Label)3344 void AssemblerX8664::emitLabelLink(Label *Label) {
3345   assert(!Label->isBound());
3346   intptr_t Position = Buffer.size();
3347   emitInt32(Label->Position);
3348   Label->linkTo(*this, Position);
3349 }
3350 
emitNearLabelLink(Label * Label)3351 void AssemblerX8664::emitNearLabelLink(Label *Label) {
3352   assert(!Label->isBound());
3353   intptr_t Position = Buffer.size();
3354   emitUint8(0);
3355   Label->nearLinkTo(*this, Position);
3356 }
3357 
emitGenericShift(int rm,Type Ty,GPRRegister reg,const Immediate & imm)3358 void AssemblerX8664::emitGenericShift(int rm, Type Ty, GPRRegister reg,
3359                                       const Immediate &imm) {
3360   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3361   // We don't assert that imm fits into 8 bits; instead, it gets masked below.
3362   // Note that we don't mask it further (e.g. to 5 bits) because we want the
3363   // same processor behavior regardless of whether it's an immediate (masked to
3364   // 8 bits) or in register cl (essentially ecx masked to 8 bits).
3365   if (Ty == IceType_i16)
3366     emitOperandSizeOverride();
3367   emitRexB(Ty, reg);
3368   if (imm.value() == 1) {
3369     emitUint8(isByteSizedArithType(Ty) ? 0xD0 : 0xD1);
3370     emitOperand(rm, AsmOperand(reg));
3371   } else {
3372     emitUint8(isByteSizedArithType(Ty) ? 0xC0 : 0xC1);
3373     static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3374     emitOperand(rm, AsmOperand(reg), OffsetFromNextInstruction);
3375     emitUint8(imm.value() & 0xFF);
3376   }
3377 }
3378 
emitGenericShift(int rm,Type Ty,const AsmOperand & operand,GPRRegister shifter)3379 void AssemblerX8664::emitGenericShift(int rm, Type Ty,
3380                                       const AsmOperand &operand,
3381                                       GPRRegister shifter) {
3382   AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3383   assert(shifter == RegX8664::Encoded_Reg_rcx);
3384   (void)shifter;
3385   if (Ty == IceType_i16)
3386     emitOperandSizeOverride();
3387   emitRexB(Ty, operand.rm());
3388   emitUint8(isByteSizedArithType(Ty) ? 0xD2 : 0xD3);
3389   emitOperand(rm, operand);
3390 }
3391 
3392 } // end of namespace X8664
3393 } // end of namespace Ice
3394