Coverage Report

Created: 2024-01-17 10:31

/src/build/lib/Target/X86/X86GenFastISel.inc
Line
Count
Source (jump to first uncovered line)
1
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2
|*                                                                            *|
3
|* "Fast" Instruction Selector for the X86 target                             *|
4
|*                                                                            *|
5
|* Automatically generated file, do not edit!                                 *|
6
|*                                                                            *|
7
\*===----------------------------------------------------------------------===*/
8
9
10
// FastEmit Immediate Predicate functions.
11
0
static bool Predicate_AndMask64(int64_t Imm) {
12
0
13
0
  return isMask_64(Imm) && !isUInt<32>(Imm);
14
0
15
0
}
16
0
static bool Predicate_BTRMask64(int64_t Imm) {
17
0
18
0
  return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
19
0
20
0
}
21
0
static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
0
23
0
  return !isInt<32>(Imm) && isPowerOf2_64(Imm);
24
0
25
0
}
26
0
static bool Predicate_i64immSExt32(int64_t Imm) {
27
0
 return isInt<32>(Imm); 
28
0
}
29
30
31
// FastEmit functions for ISD::ABS.
32
33
0
unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
34
0
  if (RetVT.SimpleTy != MVT::v16i8)
35
0
    return 0;
36
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
37
0
    return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0);
38
0
  }
39
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
40
0
    return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0);
41
0
  }
42
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
43
0
    return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0);
44
0
  }
45
0
  return 0;
46
0
}
47
48
0
unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
49
0
  if (RetVT.SimpleTy != MVT::v32i8)
50
0
    return 0;
51
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
52
0
    return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0);
53
0
  }
54
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
55
0
    return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0);
56
0
  }
57
0
  return 0;
58
0
}
59
60
0
unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0) {
61
0
  if (RetVT.SimpleTy != MVT::v64i8)
62
0
    return 0;
63
0
  if ((Subtarget->hasBWI())) {
64
0
    return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0);
65
0
  }
66
0
  return 0;
67
0
}
68
69
0
unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
70
0
  if (RetVT.SimpleTy != MVT::v8i16)
71
0
    return 0;
72
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
73
0
    return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0);
74
0
  }
75
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
76
0
    return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0);
77
0
  }
78
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
79
0
    return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0);
80
0
  }
81
0
  return 0;
82
0
}
83
84
0
unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
85
0
  if (RetVT.SimpleTy != MVT::v16i16)
86
0
    return 0;
87
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
88
0
    return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0);
89
0
  }
90
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
91
0
    return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0);
92
0
  }
93
0
  return 0;
94
0
}
95
96
0
unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
97
0
  if (RetVT.SimpleTy != MVT::v32i16)
98
0
    return 0;
99
0
  if ((Subtarget->hasBWI())) {
100
0
    return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0);
101
0
  }
102
0
  return 0;
103
0
}
104
105
0
unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
106
0
  if (RetVT.SimpleTy != MVT::v4i32)
107
0
    return 0;
108
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
109
0
    return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0);
110
0
  }
111
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
112
0
    return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0);
113
0
  }
114
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
115
0
    return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0);
116
0
  }
117
0
  return 0;
118
0
}
119
120
0
unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
121
0
  if (RetVT.SimpleTy != MVT::v8i32)
122
0
    return 0;
123
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
124
0
    return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0);
125
0
  }
126
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
127
0
    return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0);
128
0
  }
129
0
  return 0;
130
0
}
131
132
0
unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
133
0
  if (RetVT.SimpleTy != MVT::v16i32)
134
0
    return 0;
135
0
  if ((Subtarget->hasAVX512())) {
136
0
    return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0);
137
0
  }
138
0
  return 0;
139
0
}
140
141
0
unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
142
0
  if (RetVT.SimpleTy != MVT::v2i64)
143
0
    return 0;
144
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
145
0
    return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0);
146
0
  }
147
0
  return 0;
148
0
}
149
150
0
unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
151
0
  if (RetVT.SimpleTy != MVT::v4i64)
152
0
    return 0;
153
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154
0
    return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0);
155
0
  }
156
0
  return 0;
157
0
}
158
159
0
unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
160
0
  if (RetVT.SimpleTy != MVT::v8i64)
161
0
    return 0;
162
0
  if ((Subtarget->hasAVX512())) {
163
0
    return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0);
164
0
  }
165
0
  return 0;
166
0
}
167
168
0
unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) {
169
0
  switch (VT.SimpleTy) {
170
0
  case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
171
0
  case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0);
172
0
  case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0);
173
0
  case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
174
0
  case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0);
175
0
  case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0);
176
0
  case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
177
0
  case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0);
178
0
  case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0);
179
0
  case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
180
0
  case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0);
181
0
  case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0);
182
0
  default: return 0;
183
0
  }
184
0
}
185
186
// FastEmit functions for ISD::ANY_EXTEND.
187
188
0
unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
189
0
  if (RetVT.SimpleTy != MVT::i32)
190
0
    return 0;
191
0
  return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0);
192
0
}
193
194
0
unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
195
0
  switch (VT.SimpleTy) {
196
0
  case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0);
197
0
  default: return 0;
198
0
  }
199
0
}
200
201
// FastEmit functions for ISD::BITCAST.
202
203
0
unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
204
0
  if (RetVT.SimpleTy != MVT::f32)
205
0
    return 0;
206
0
  if ((Subtarget->hasAVX512())) {
207
0
    return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0);
208
0
  }
209
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
210
0
    return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0);
211
0
  }
212
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
213
0
    return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0);
214
0
  }
215
0
  return 0;
216
0
}
217
218
0
unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0) {
219
0
  if ((Subtarget->hasAVX512())) {
220
0
    return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0);
221
0
  }
222
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
223
0
    return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0);
224
0
  }
225
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
226
0
    return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0);
227
0
  }
228
0
  return 0;
229
0
}
230
231
0
unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0) {
232
0
  if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
233
0
    return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0);
234
0
  }
235
0
  return 0;
236
0
}
237
238
0
unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0) {
239
0
switch (RetVT.SimpleTy) {
240
0
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0);
241
0
  case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0);
242
0
  default: return 0;
243
0
}
244
0
}
245
246
0
unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0) {
247
0
  if (RetVT.SimpleTy != MVT::i32)
248
0
    return 0;
249
0
  if ((Subtarget->hasAVX512())) {
250
0
    return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0);
251
0
  }
252
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
253
0
    return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0);
254
0
  }
255
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
256
0
    return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0);
257
0
  }
258
0
  return 0;
259
0
}
260
261
0
unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0) {
262
0
  if ((Subtarget->hasAVX512())) {
263
0
    return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0);
264
0
  }
265
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
266
0
    return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0);
267
0
  }
268
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
269
0
    return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0);
270
0
  }
271
0
  return 0;
272
0
}
273
274
0
unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0) {
275
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
276
0
    return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0);
277
0
  }
278
0
  return 0;
279
0
}
280
281
0
unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) {
282
0
switch (RetVT.SimpleTy) {
283
0
  case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0);
284
0
  case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0);
285
0
  default: return 0;
286
0
}
287
0
}
288
289
0
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0) {
290
0
  if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) {
291
0
    return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0);
292
0
  }
293
0
  return 0;
294
0
}
295
296
0
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0) {
297
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
298
0
    return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0);
299
0
  }
300
0
  return 0;
301
0
}
302
303
0
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
304
0
switch (RetVT.SimpleTy) {
305
0
  case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0);
306
0
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0);
307
0
  default: return 0;
308
0
}
309
0
}
310
311
0
unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
312
0
  switch (VT.SimpleTy) {
313
0
  case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
314
0
  case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0);
315
0
  case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
316
0
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
317
0
  case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0);
318
0
  default: return 0;
319
0
  }
320
0
}
321
322
// FastEmit functions for ISD::BRIND.
323
324
0
unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) {
325
0
  if (RetVT.SimpleTy != MVT::isVoid)
326
0
    return 0;
327
0
  if ((!Subtarget->is64Bit())) {
328
0
    return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0);
329
0
  }
330
0
  return 0;
331
0
}
332
333
0
unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
334
0
  if (RetVT.SimpleTy != MVT::isVoid)
335
0
    return 0;
336
0
  if ((!Subtarget->is64Bit())) {
337
0
    return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0);
338
0
  }
339
0
  return 0;
340
0
}
341
342
0
unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) {
343
0
  if (RetVT.SimpleTy != MVT::isVoid)
344
0
    return 0;
345
0
  if ((Subtarget->is64Bit())) {
346
0
    return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0);
347
0
  }
348
0
  return 0;
349
0
}
350
351
0
unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
352
0
  switch (VT.SimpleTy) {
353
0
  case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0);
354
0
  case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
355
0
  case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
356
0
  default: return 0;
357
0
  }
358
0
}
359
360
// FastEmit functions for ISD::BSWAP.
361
362
0
unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) {
363
0
  if (RetVT.SimpleTy != MVT::i32)
364
0
    return 0;
365
0
  return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0);
366
0
}
367
368
0
unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0) {
369
0
  if (RetVT.SimpleTy != MVT::i64)
370
0
    return 0;
371
0
  return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0);
372
0
}
373
374
0
unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) {
375
0
  switch (VT.SimpleTy) {
376
0
  case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
377
0
  case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
378
0
  default: return 0;
379
0
  }
380
0
}
381
382
// FastEmit functions for ISD::CTLZ.
383
384
0
unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0) {
385
0
  if (RetVT.SimpleTy != MVT::i16)
386
0
    return 0;
387
0
  if ((Subtarget->hasLZCNT())) {
388
0
    return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0);
389
0
  }
390
0
  return 0;
391
0
}
392
393
0
unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
394
0
  if (RetVT.SimpleTy != MVT::i32)
395
0
    return 0;
396
0
  if ((Subtarget->hasLZCNT())) {
397
0
    return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0);
398
0
  }
399
0
  return 0;
400
0
}
401
402
0
unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
403
0
  if (RetVT.SimpleTy != MVT::i64)
404
0
    return 0;
405
0
  if ((Subtarget->hasLZCNT())) {
406
0
    return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0);
407
0
  }
408
0
  return 0;
409
0
}
410
411
0
unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
412
0
  if (RetVT.SimpleTy != MVT::v4i32)
413
0
    return 0;
414
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
415
0
    return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0);
416
0
  }
417
0
  return 0;
418
0
}
419
420
0
unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
421
0
  if (RetVT.SimpleTy != MVT::v8i32)
422
0
    return 0;
423
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
424
0
    return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0);
425
0
  }
426
0
  return 0;
427
0
}
428
429
0
unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
430
0
  if (RetVT.SimpleTy != MVT::v16i32)
431
0
    return 0;
432
0
  if ((Subtarget->hasCDI())) {
433
0
    return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0);
434
0
  }
435
0
  return 0;
436
0
}
437
438
0
unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
439
0
  if (RetVT.SimpleTy != MVT::v2i64)
440
0
    return 0;
441
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
442
0
    return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0);
443
0
  }
444
0
  return 0;
445
0
}
446
447
0
unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
448
0
  if (RetVT.SimpleTy != MVT::v4i64)
449
0
    return 0;
450
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
451
0
    return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0);
452
0
  }
453
0
  return 0;
454
0
}
455
456
0
unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
457
0
  if (RetVT.SimpleTy != MVT::v8i64)
458
0
    return 0;
459
0
  if ((Subtarget->hasCDI())) {
460
0
    return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0);
461
0
  }
462
0
  return 0;
463
0
}
464
465
0
unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) {
466
0
  switch (VT.SimpleTy) {
467
0
  case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0);
468
0
  case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
469
0
  case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
470
0
  case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
471
0
  case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0);
472
0
  case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0);
473
0
  case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0);
474
0
  case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0);
475
0
  case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0);
476
0
  default: return 0;
477
0
  }
478
0
}
479
480
// FastEmit functions for ISD::CTPOP.
481
482
0
unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0) {
483
0
  if (RetVT.SimpleTy != MVT::i16)
484
0
    return 0;
485
0
  if ((Subtarget->hasPOPCNT())) {
486
0
    return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0);
487
0
  }
488
0
  return 0;
489
0
}
490
491
0
unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0) {
492
0
  if (RetVT.SimpleTy != MVT::i32)
493
0
    return 0;
494
0
  if ((Subtarget->hasPOPCNT())) {
495
0
    return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0);
496
0
  }
497
0
  return 0;
498
0
}
499
500
0
unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0) {
501
0
  if (RetVT.SimpleTy != MVT::i64)
502
0
    return 0;
503
0
  if ((Subtarget->hasPOPCNT())) {
504
0
    return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0);
505
0
  }
506
0
  return 0;
507
0
}
508
509
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
510
0
  if (RetVT.SimpleTy != MVT::v16i8)
511
0
    return 0;
512
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
513
0
    return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0);
514
0
  }
515
0
  return 0;
516
0
}
517
518
0
unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
519
0
  if (RetVT.SimpleTy != MVT::v32i8)
520
0
    return 0;
521
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
522
0
    return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0);
523
0
  }
524
0
  return 0;
525
0
}
526
527
0
unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0) {
528
0
  if (RetVT.SimpleTy != MVT::v64i8)
529
0
    return 0;
530
0
  if ((Subtarget->hasBITALG())) {
531
0
    return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0);
532
0
  }
533
0
  return 0;
534
0
}
535
536
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
537
0
  if (RetVT.SimpleTy != MVT::v8i16)
538
0
    return 0;
539
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
540
0
    return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0);
541
0
  }
542
0
  return 0;
543
0
}
544
545
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
546
0
  if (RetVT.SimpleTy != MVT::v16i16)
547
0
    return 0;
548
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
549
0
    return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0);
550
0
  }
551
0
  return 0;
552
0
}
553
554
0
unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
555
0
  if (RetVT.SimpleTy != MVT::v32i16)
556
0
    return 0;
557
0
  if ((Subtarget->hasBITALG())) {
558
0
    return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0);
559
0
  }
560
0
  return 0;
561
0
}
562
563
0
unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
564
0
  if (RetVT.SimpleTy != MVT::v4i32)
565
0
    return 0;
566
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
567
0
    return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0);
568
0
  }
569
0
  return 0;
570
0
}
571
572
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
573
0
  if (RetVT.SimpleTy != MVT::v8i32)
574
0
    return 0;
575
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
576
0
    return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0);
577
0
  }
578
0
  return 0;
579
0
}
580
581
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
582
0
  if (RetVT.SimpleTy != MVT::v16i32)
583
0
    return 0;
584
0
  if ((Subtarget->hasVPOPCNTDQ())) {
585
0
    return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0);
586
0
  }
587
0
  return 0;
588
0
}
589
590
0
unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
591
0
  if (RetVT.SimpleTy != MVT::v2i64)
592
0
    return 0;
593
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
594
0
    return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0);
595
0
  }
596
0
  return 0;
597
0
}
598
599
0
unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
600
0
  if (RetVT.SimpleTy != MVT::v4i64)
601
0
    return 0;
602
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
603
0
    return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0);
604
0
  }
605
0
  return 0;
606
0
}
607
608
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
609
0
  if (RetVT.SimpleTy != MVT::v8i64)
610
0
    return 0;
611
0
  if ((Subtarget->hasVPOPCNTDQ())) {
612
0
    return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0);
613
0
  }
614
0
  return 0;
615
0
}
616
617
0
unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) {
618
0
  switch (VT.SimpleTy) {
619
0
  case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0);
620
0
  case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
621
0
  case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
622
0
  case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
623
0
  case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0);
624
0
  case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0);
625
0
  case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0);
626
0
  case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0);
627
0
  case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0);
628
0
  case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0);
629
0
  case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0);
630
0
  case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0);
631
0
  case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0);
632
0
  case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0);
633
0
  case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0);
634
0
  default: return 0;
635
0
  }
636
0
}
637
638
// FastEmit functions for ISD::CTTZ.
639
640
0
unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0) {
641
0
  if (RetVT.SimpleTy != MVT::i16)
642
0
    return 0;
643
0
  if ((Subtarget->hasBMI())) {
644
0
    return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0);
645
0
  }
646
0
  return 0;
647
0
}
648
649
0
unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
650
0
  if (RetVT.SimpleTy != MVT::i32)
651
0
    return 0;
652
0
  if ((Subtarget->hasBMI())) {
653
0
    return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0);
654
0
  }
655
0
  return 0;
656
0
}
657
658
0
unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
659
0
  if (RetVT.SimpleTy != MVT::i64)
660
0
    return 0;
661
0
  if ((Subtarget->hasBMI())) {
662
0
    return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0);
663
0
  }
664
0
  return 0;
665
0
}
666
667
0
unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0) {
668
0
  switch (VT.SimpleTy) {
669
0
  case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0);
670
0
  case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
671
0
  case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
672
0
  default: return 0;
673
0
  }
674
0
}
675
676
// FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
677
678
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0) {
679
0
  if (RetVT.SimpleTy != MVT::i16)
680
0
    return 0;
681
0
  return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0);
682
0
}
683
684
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0) {
685
0
  if (RetVT.SimpleTy != MVT::i32)
686
0
    return 0;
687
0
  return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0);
688
0
}
689
690
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0) {
691
0
  if (RetVT.SimpleTy != MVT::i64)
692
0
    return 0;
693
0
  return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0);
694
0
}
695
696
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0) {
697
0
  switch (VT.SimpleTy) {
698
0
  case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0);
699
0
  case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0);
700
0
  case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0);
701
0
  default: return 0;
702
0
  }
703
0
}
704
705
// FastEmit functions for ISD::FABS.
706
707
0
unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) {
708
0
  if (RetVT.SimpleTy != MVT::f32)
709
0
    return 0;
710
0
  if ((!Subtarget->hasSSE1())) {
711
0
    return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0);
712
0
  }
713
0
  return 0;
714
0
}
715
716
0
unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) {
717
0
  if (RetVT.SimpleTy != MVT::f64)
718
0
    return 0;
719
0
  if ((!Subtarget->hasSSE2())) {
720
0
    return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0);
721
0
  }
722
0
  return 0;
723
0
}
724
725
0
unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0) {
726
0
  if (RetVT.SimpleTy != MVT::f80)
727
0
    return 0;
728
0
  if ((Subtarget->hasX87())) {
729
0
    return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0);
730
0
  }
731
0
  return 0;
732
0
}
733
734
0
unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) {
735
0
  switch (VT.SimpleTy) {
736
0
  case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
737
0
  case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
738
0
  case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0);
739
0
  default: return 0;
740
0
  }
741
0
}
742
743
// FastEmit functions for ISD::FNEG.
744
745
0
unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) {
746
0
  if (RetVT.SimpleTy != MVT::f32)
747
0
    return 0;
748
0
  if ((!Subtarget->hasSSE1())) {
749
0
    return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0);
750
0
  }
751
0
  return 0;
752
0
}
753
754
0
unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) {
755
0
  if (RetVT.SimpleTy != MVT::f64)
756
0
    return 0;
757
0
  if ((!Subtarget->hasSSE2())) {
758
0
    return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0);
759
0
  }
760
0
  return 0;
761
0
}
762
763
0
unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0) {
764
0
  if (RetVT.SimpleTy != MVT::f80)
765
0
    return 0;
766
0
  if ((Subtarget->hasX87())) {
767
0
    return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0);
768
0
  }
769
0
  return 0;
770
0
}
771
772
0
unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) {
773
0
  switch (VT.SimpleTy) {
774
0
  case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
775
0
  case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
776
0
  case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0);
777
0
  default: return 0;
778
0
  }
779
0
}
780
781
// FastEmit functions for ISD::FP_EXTEND.
782
783
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
784
0
  if (RetVT.SimpleTy != MVT::f64)
785
0
    return 0;
786
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
787
0
    return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0);
788
0
  }
789
0
  return 0;
790
0
}
791
792
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) {
793
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
794
0
    return fastEmitInst_r(X86::VCVTPH2PSXZ256rr, &X86::VR256XRegClass, Op0);
795
0
  }
796
0
  return 0;
797
0
}
798
799
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) {
800
0
  if ((Subtarget->hasFP16())) {
801
0
    return fastEmitInst_r(X86::VCVTPH2PDZrr, &X86::VR512RegClass, Op0);
802
0
  }
803
0
  return 0;
804
0
}
805
806
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
807
0
switch (RetVT.SimpleTy) {
808
0
  case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
809
0
  case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
810
0
  default: return 0;
811
0
}
812
0
}
813
814
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
815
0
  if (RetVT.SimpleTy != MVT::v16f32)
816
0
    return 0;
817
0
  if ((Subtarget->hasFP16())) {
818
0
    return fastEmitInst_r(X86::VCVTPH2PSXZrr, &X86::VR512RegClass, Op0);
819
0
  }
820
0
  return 0;
821
0
}
822
823
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
824
0
  if (RetVT.SimpleTy != MVT::v4f64)
825
0
    return 0;
826
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
827
0
    return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0);
828
0
  }
829
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
830
0
    return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0);
831
0
  }
832
0
  return 0;
833
0
}
834
835
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
836
0
  if (RetVT.SimpleTy != MVT::v8f64)
837
0
    return 0;
838
0
  if ((Subtarget->hasAVX512())) {
839
0
    return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0);
840
0
  }
841
0
  return 0;
842
0
}
843
844
0
unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
845
0
  switch (VT.SimpleTy) {
846
0
  case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
847
0
  case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
848
0
  case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
849
0
  case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
850
0
  case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
851
0
  default: return 0;
852
0
  }
853
0
}
854
855
// FastEmit functions for ISD::FP_ROUND.
856
857
0
unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
858
0
  if (RetVT.SimpleTy != MVT::f32)
859
0
    return 0;
860
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
861
0
    return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0);
862
0
  }
863
0
  return 0;
864
0
}
865
866
0
unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
867
0
  switch (VT.SimpleTy) {
868
0
  case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
869
0
  default: return 0;
870
0
  }
871
0
}
872
873
// FastEmit functions for ISD::FP_TO_SINT.
874
875
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
876
0
  if ((Subtarget->hasFP16())) {
877
0
    return fastEmitInst_r(X86::VCVTTSH2SIZrr, &X86::GR32RegClass, Op0);
878
0
  }
879
0
  return 0;
880
0
}
881
882
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
883
0
  if ((Subtarget->hasFP16())) {
884
0
    return fastEmitInst_r(X86::VCVTTSH2SI64Zrr, &X86::GR64RegClass, Op0);
885
0
  }
886
0
  return 0;
887
0
}
888
889
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
890
0
switch (RetVT.SimpleTy) {
891
0
  case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
892
0
  case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
893
0
  default: return 0;
894
0
}
895
0
}
896
897
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
898
0
  if ((Subtarget->hasAVX512())) {
899
0
    return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0);
900
0
  }
901
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
902
0
    return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0);
903
0
  }
904
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
905
0
    return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0);
906
0
  }
907
0
  return 0;
908
0
}
909
910
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
911
0
  if ((Subtarget->hasAVX512())) {
912
0
    return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0);
913
0
  }
914
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
915
0
    return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0);
916
0
  }
917
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
918
0
    return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0);
919
0
  }
920
0
  return 0;
921
0
}
922
923
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
924
0
switch (RetVT.SimpleTy) {
925
0
  case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
926
0
  case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
927
0
  default: return 0;
928
0
}
929
0
}
930
931
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
932
0
  if ((Subtarget->hasAVX512())) {
933
0
    return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0);
934
0
  }
935
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
936
0
    return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0);
937
0
  }
938
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
939
0
    return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0);
940
0
  }
941
0
  return 0;
942
0
}
943
944
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
945
0
  if ((Subtarget->hasAVX512())) {
946
0
    return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0);
947
0
  }
948
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
949
0
    return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0);
950
0
  }
951
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
952
0
    return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0);
953
0
  }
954
0
  return 0;
955
0
}
956
957
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
958
0
switch (RetVT.SimpleTy) {
959
0
  case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
960
0
  case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
961
0
  default: return 0;
962
0
}
963
0
}
964
965
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
966
0
  if (RetVT.SimpleTy != MVT::v4i32)
967
0
    return 0;
968
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
969
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0);
970
0
  }
971
0
  return 0;
972
0
}
973
974
0
unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
975
0
  switch (VT.SimpleTy) {
976
0
  case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
977
0
  case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
978
0
  case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
979
0
  case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
980
0
  default: return 0;
981
0
  }
982
0
}
983
984
// FastEmit functions for ISD::FP_TO_UINT.
985
986
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
987
0
  if ((Subtarget->hasFP16())) {
988
0
    return fastEmitInst_r(X86::VCVTTSH2USIZrr, &X86::GR32RegClass, Op0);
989
0
  }
990
0
  return 0;
991
0
}
992
993
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
994
0
  if ((Subtarget->hasFP16())) {
995
0
    return fastEmitInst_r(X86::VCVTTSH2USI64Zrr, &X86::GR64RegClass, Op0);
996
0
  }
997
0
  return 0;
998
0
}
999
1000
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
1001
0
switch (RetVT.SimpleTy) {
1002
0
  case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
1003
0
  case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
1004
0
  default: return 0;
1005
0
}
1006
0
}
1007
1008
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
1009
0
  if ((Subtarget->hasAVX512())) {
1010
0
    return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0);
1011
0
  }
1012
0
  return 0;
1013
0
}
1014
1015
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
1016
0
  if ((Subtarget->hasAVX512())) {
1017
0
    return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0);
1018
0
  }
1019
0
  return 0;
1020
0
}
1021
1022
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1023
0
switch (RetVT.SimpleTy) {
1024
0
  case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
1025
0
  case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
1026
0
  default: return 0;
1027
0
}
1028
0
}
1029
1030
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
1031
0
  if ((Subtarget->hasAVX512())) {
1032
0
    return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0);
1033
0
  }
1034
0
  return 0;
1035
0
}
1036
1037
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
1038
0
  if ((Subtarget->hasAVX512())) {
1039
0
    return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0);
1040
0
  }
1041
0
  return 0;
1042
0
}
1043
1044
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1045
0
switch (RetVT.SimpleTy) {
1046
0
  case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
1047
0
  case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
1048
0
  default: return 0;
1049
0
}
1050
0
}
1051
1052
0
unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1053
0
  switch (VT.SimpleTy) {
1054
0
  case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
1055
0
  case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
1056
0
  case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
1057
0
  default: return 0;
1058
0
  }
1059
0
}
1060
1061
// FastEmit functions for ISD::FSQRT.
1062
1063
0
unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1064
0
  if (RetVT.SimpleTy != MVT::f32)
1065
0
    return 0;
1066
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1067
0
    return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0);
1068
0
  }
1069
0
  if ((!Subtarget->hasSSE1())) {
1070
0
    return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0);
1071
0
  }
1072
0
  return 0;
1073
0
}
1074
1075
0
unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1076
0
  if (RetVT.SimpleTy != MVT::f64)
1077
0
    return 0;
1078
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1079
0
    return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0);
1080
0
  }
1081
0
  if ((!Subtarget->hasSSE2())) {
1082
0
    return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0);
1083
0
  }
1084
0
  return 0;
1085
0
}
1086
1087
0
unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) {
1088
0
  if (RetVT.SimpleTy != MVT::f80)
1089
0
    return 0;
1090
0
  if ((Subtarget->hasX87())) {
1091
0
    return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0);
1092
0
  }
1093
0
  return 0;
1094
0
}
1095
1096
0
unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1097
0
  if (RetVT.SimpleTy != MVT::v8f16)
1098
0
    return 0;
1099
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1100
0
    return fastEmitInst_r(X86::VSQRTPHZ128r, &X86::VR128XRegClass, Op0);
1101
0
  }
1102
0
  return 0;
1103
0
}
1104
1105
0
unsigned fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
1106
0
  if (RetVT.SimpleTy != MVT::v16f16)
1107
0
    return 0;
1108
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1109
0
    return fastEmitInst_r(X86::VSQRTPHZ256r, &X86::VR256XRegClass, Op0);
1110
0
  }
1111
0
  return 0;
1112
0
}
1113
1114
0
unsigned fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
1115
0
  if (RetVT.SimpleTy != MVT::v32f16)
1116
0
    return 0;
1117
0
  if ((Subtarget->hasFP16())) {
1118
0
    return fastEmitInst_r(X86::VSQRTPHZr, &X86::VR512RegClass, Op0);
1119
0
  }
1120
0
  return 0;
1121
0
}
1122
1123
0
unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1124
0
  if (RetVT.SimpleTy != MVT::v4f32)
1125
0
    return 0;
1126
0
  if ((Subtarget->hasVLX())) {
1127
0
    return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0);
1128
0
  }
1129
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1130
0
    return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0);
1131
0
  }
1132
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1133
0
    return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0);
1134
0
  }
1135
0
  return 0;
1136
0
}
1137
1138
0
unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
1139
0
  if (RetVT.SimpleTy != MVT::v8f32)
1140
0
    return 0;
1141
0
  if ((Subtarget->hasVLX())) {
1142
0
    return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0);
1143
0
  }
1144
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1145
0
    return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0);
1146
0
  }
1147
0
  return 0;
1148
0
}
1149
1150
0
unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
1151
0
  if (RetVT.SimpleTy != MVT::v16f32)
1152
0
    return 0;
1153
0
  if ((Subtarget->hasAVX512())) {
1154
0
    return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0);
1155
0
  }
1156
0
  return 0;
1157
0
}
1158
1159
0
unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1160
0
  if (RetVT.SimpleTy != MVT::v2f64)
1161
0
    return 0;
1162
0
  if ((Subtarget->hasVLX())) {
1163
0
    return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0);
1164
0
  }
1165
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1166
0
    return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0);
1167
0
  }
1168
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1169
0
    return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0);
1170
0
  }
1171
0
  return 0;
1172
0
}
1173
1174
0
unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
1175
0
  if (RetVT.SimpleTy != MVT::v4f64)
1176
0
    return 0;
1177
0
  if ((Subtarget->hasVLX())) {
1178
0
    return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0);
1179
0
  }
1180
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1181
0
    return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0);
1182
0
  }
1183
0
  return 0;
1184
0
}
1185
1186
0
unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
1187
0
  if (RetVT.SimpleTy != MVT::v8f64)
1188
0
    return 0;
1189
0
  if ((Subtarget->hasAVX512())) {
1190
0
    return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0);
1191
0
  }
1192
0
  return 0;
1193
0
}
1194
1195
0
unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
1196
0
  switch (VT.SimpleTy) {
1197
0
  case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
1198
0
  case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
1199
0
  case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0);
1200
0
  case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
1201
0
  case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0);
1202
0
  case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0);
1203
0
  case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
1204
0
  case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0);
1205
0
  case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0);
1206
0
  case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
1207
0
  case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0);
1208
0
  case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0);
1209
0
  default: return 0;
1210
0
  }
1211
0
}
1212
1213
// FastEmit functions for ISD::LLRINT.
1214
1215
0
unsigned fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1216
0
  if (RetVT.SimpleTy != MVT::i64)
1217
0
    return 0;
1218
0
  if ((Subtarget->hasAVX512())) {
1219
0
    return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0);
1220
0
  }
1221
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1222
0
    return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0);
1223
0
  }
1224
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1225
0
    return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0);
1226
0
  }
1227
0
  return 0;
1228
0
}
1229
1230
0
unsigned fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1231
0
  if (RetVT.SimpleTy != MVT::i64)
1232
0
    return 0;
1233
0
  if ((Subtarget->hasAVX512())) {
1234
0
    return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0);
1235
0
  }
1236
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1237
0
    return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0);
1238
0
  }
1239
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1240
0
    return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0);
1241
0
  }
1242
0
  return 0;
1243
0
}
1244
1245
0
unsigned fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1246
0
  switch (VT.SimpleTy) {
1247
0
  case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0);
1248
0
  case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0);
1249
0
  default: return 0;
1250
0
  }
1251
0
}
1252
1253
// FastEmit functions for ISD::LRINT.
1254
1255
0
unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(unsigned Op0) {
1256
0
  if ((Subtarget->hasAVX512())) {
1257
0
    return fastEmitInst_r(X86::VCVTSS2SIZrr, &X86::GR32RegClass, Op0);
1258
0
  }
1259
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1260
0
    return fastEmitInst_r(X86::CVTSS2SIrr, &X86::GR32RegClass, Op0);
1261
0
  }
1262
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1263
0
    return fastEmitInst_r(X86::VCVTSS2SIrr, &X86::GR32RegClass, Op0);
1264
0
  }
1265
0
  return 0;
1266
0
}
1267
1268
0
unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(unsigned Op0) {
1269
0
  if ((Subtarget->hasAVX512())) {
1270
0
    return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0);
1271
0
  }
1272
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1273
0
    return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0);
1274
0
  }
1275
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1276
0
    return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0);
1277
0
  }
1278
0
  return 0;
1279
0
}
1280
1281
0
unsigned fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
1282
0
switch (RetVT.SimpleTy) {
1283
0
  case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0);
1284
0
  case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0);
1285
0
  default: return 0;
1286
0
}
1287
0
}
1288
1289
0
unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(unsigned Op0) {
1290
0
  if ((Subtarget->hasAVX512())) {
1291
0
    return fastEmitInst_r(X86::VCVTSD2SIZrr, &X86::GR32RegClass, Op0);
1292
0
  }
1293
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1294
0
    return fastEmitInst_r(X86::CVTSD2SIrr, &X86::GR32RegClass, Op0);
1295
0
  }
1296
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1297
0
    return fastEmitInst_r(X86::VCVTSD2SIrr, &X86::GR32RegClass, Op0);
1298
0
  }
1299
0
  return 0;
1300
0
}
1301
1302
0
unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(unsigned Op0) {
1303
0
  if ((Subtarget->hasAVX512())) {
1304
0
    return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0);
1305
0
  }
1306
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1307
0
    return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0);
1308
0
  }
1309
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1310
0
    return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0);
1311
0
  }
1312
0
  return 0;
1313
0
}
1314
1315
0
unsigned fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
1316
0
switch (RetVT.SimpleTy) {
1317
0
  case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0);
1318
0
  case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0);
1319
0
  default: return 0;
1320
0
}
1321
0
}
1322
1323
0
unsigned fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
1324
0
  switch (VT.SimpleTy) {
1325
0
  case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0);
1326
0
  case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0);
1327
0
  default: return 0;
1328
0
  }
1329
0
}
1330
1331
// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1332
1333
0
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0) {
1334
0
  if (RetVT.SimpleTy != MVT::v4i32)
1335
0
    return 0;
1336
0
  if ((Subtarget->hasAVX512())) {
1337
0
    return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0);
1338
0
  }
1339
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1340
0
    return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0);
1341
0
  }
1342
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1343
0
    return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0);
1344
0
  }
1345
0
  return 0;
1346
0
}
1347
1348
0
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0) {
1349
0
  if (RetVT.SimpleTy != MVT::v2i64)
1350
0
    return 0;
1351
0
  if ((Subtarget->hasAVX512())) {
1352
0
    return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0);
1353
0
  }
1354
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1355
0
    return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0);
1356
0
  }
1357
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1358
0
    return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0);
1359
0
  }
1360
0
  return 0;
1361
0
}
1362
1363
0
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0) {
1364
0
  switch (VT.SimpleTy) {
1365
0
  case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0);
1366
0
  case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0);
1367
0
  default: return 0;
1368
0
  }
1369
0
}
1370
1371
// FastEmit functions for ISD::SIGN_EXTEND.
1372
1373
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0) {
1374
0
  return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0);
1375
0
}
1376
1377
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0) {
1378
0
  return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0);
1379
0
}
1380
1381
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
1382
0
switch (RetVT.SimpleTy) {
1383
0
  case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0);
1384
0
  case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0);
1385
0
  default: return 0;
1386
0
}
1387
0
}
1388
1389
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0) {
1390
0
  return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0);
1391
0
}
1392
1393
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0) {
1394
0
  return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0);
1395
0
}
1396
1397
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) {
1398
0
switch (RetVT.SimpleTy) {
1399
0
  case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0);
1400
0
  case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0);
1401
0
  default: return 0;
1402
0
}
1403
0
}
1404
1405
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0) {
1406
0
  if (RetVT.SimpleTy != MVT::i64)
1407
0
    return 0;
1408
0
  if ((Subtarget->is64Bit())) {
1409
0
    return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0);
1410
0
  }
1411
0
  return 0;
1412
0
}
1413
1414
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0) {
1415
0
  if (RetVT.SimpleTy != MVT::v2i64)
1416
0
    return 0;
1417
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1418
0
    return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0);
1419
0
  }
1420
0
  return 0;
1421
0
}
1422
1423
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0) {
1424
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1425
0
    return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0);
1426
0
  }
1427
0
  return 0;
1428
0
}
1429
1430
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0) {
1431
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1432
0
    return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0);
1433
0
  }
1434
0
  return 0;
1435
0
}
1436
1437
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0) {
1438
0
switch (RetVT.SimpleTy) {
1439
0
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0);
1440
0
  case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0);
1441
0
  default: return 0;
1442
0
}
1443
0
}
1444
1445
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0) {
1446
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1447
0
    return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0);
1448
0
  }
1449
0
  return 0;
1450
0
}
1451
1452
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0) {
1453
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1454
0
    return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0);
1455
0
  }
1456
0
  return 0;
1457
0
}
1458
1459
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0) {
1460
0
  if ((Subtarget->hasDQI())) {
1461
0
    return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0);
1462
0
  }
1463
0
  return 0;
1464
0
}
1465
1466
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0) {
1467
0
switch (RetVT.SimpleTy) {
1468
0
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0);
1469
0
  case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0);
1470
0
  case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0);
1471
0
  default: return 0;
1472
0
}
1473
0
}
1474
1475
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0) {
1476
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1477
0
    return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0);
1478
0
  }
1479
0
  return 0;
1480
0
}
1481
1482
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0) {
1483
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1484
0
    return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0);
1485
0
  }
1486
0
  return 0;
1487
0
}
1488
1489
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0) {
1490
0
  if ((Subtarget->hasDQI())) {
1491
0
    return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0);
1492
0
  }
1493
0
  return 0;
1494
0
}
1495
1496
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0) {
1497
0
switch (RetVT.SimpleTy) {
1498
0
  case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0);
1499
0
  case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0);
1500
0
  case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0);
1501
0
  default: return 0;
1502
0
}
1503
0
}
1504
1505
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0) {
1506
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1507
0
    return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0);
1508
0
  }
1509
0
  return 0;
1510
0
}
1511
1512
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0) {
1513
0
  if ((Subtarget->hasBWI())) {
1514
0
    return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0);
1515
0
  }
1516
0
  return 0;
1517
0
}
1518
1519
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0) {
1520
0
switch (RetVT.SimpleTy) {
1521
0
  case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0);
1522
0
  case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0);
1523
0
  default: return 0;
1524
0
}
1525
0
}
1526
1527
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0) {
1528
0
  if (RetVT.SimpleTy != MVT::v64i8)
1529
0
    return 0;
1530
0
  if ((Subtarget->hasBWI())) {
1531
0
    return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0);
1532
0
  }
1533
0
  return 0;
1534
0
}
1535
1536
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) {
1537
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1538
0
    return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0);
1539
0
  }
1540
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1541
0
    return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0);
1542
0
  }
1543
0
  return 0;
1544
0
}
1545
1546
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) {
1547
0
  if ((Subtarget->hasAVX512())) {
1548
0
    return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0);
1549
0
  }
1550
0
  return 0;
1551
0
}
1552
1553
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1554
0
switch (RetVT.SimpleTy) {
1555
0
  case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
1556
0
  case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
1557
0
  default: return 0;
1558
0
}
1559
0
}
1560
1561
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
1562
0
  if (RetVT.SimpleTy != MVT::v32i16)
1563
0
    return 0;
1564
0
  if ((Subtarget->hasBWI())) {
1565
0
    return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0);
1566
0
  }
1567
0
  return 0;
1568
0
}
1569
1570
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) {
1571
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1572
0
    return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0);
1573
0
  }
1574
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1575
0
    return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0);
1576
0
  }
1577
0
  return 0;
1578
0
}
1579
1580
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) {
1581
0
  if ((Subtarget->hasAVX512())) {
1582
0
    return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0);
1583
0
  }
1584
0
  return 0;
1585
0
}
1586
1587
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1588
0
switch (RetVT.SimpleTy) {
1589
0
  case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
1590
0
  case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
1591
0
  default: return 0;
1592
0
}
1593
0
}
1594
1595
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
1596
0
  if (RetVT.SimpleTy != MVT::v16i32)
1597
0
    return 0;
1598
0
  if ((Subtarget->hasAVX512())) {
1599
0
    return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0);
1600
0
  }
1601
0
  return 0;
1602
0
}
1603
1604
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1605
0
  if (RetVT.SimpleTy != MVT::v4i64)
1606
0
    return 0;
1607
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1608
0
    return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0);
1609
0
  }
1610
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1611
0
    return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0);
1612
0
  }
1613
0
  return 0;
1614
0
}
1615
1616
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
1617
0
  if (RetVT.SimpleTy != MVT::v8i64)
1618
0
    return 0;
1619
0
  if ((Subtarget->hasAVX512())) {
1620
0
    return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0);
1621
0
  }
1622
0
  return 0;
1623
0
}
1624
1625
0
unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
1626
0
  switch (VT.SimpleTy) {
1627
0
  case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0);
1628
0
  case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0);
1629
0
  case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0);
1630
0
  case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0);
1631
0
  case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0);
1632
0
  case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0);
1633
0
  case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0);
1634
0
  case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0);
1635
0
  case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0);
1636
0
  case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0);
1637
0
  case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0);
1638
0
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0);
1639
0
  case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0);
1640
0
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0);
1641
0
  case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0);
1642
0
  default: return 0;
1643
0
  }
1644
0
}
1645
1646
// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1647
1648
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
1649
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1650
0
    return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0);
1651
0
  }
1652
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1653
0
    return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0);
1654
0
  }
1655
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1656
0
    return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0);
1657
0
  }
1658
0
  return 0;
1659
0
}
1660
1661
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
1662
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1663
0
    return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0);
1664
0
  }
1665
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1666
0
    return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0);
1667
0
  }
1668
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1669
0
    return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0);
1670
0
  }
1671
0
  return 0;
1672
0
}
1673
1674
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) {
1675
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1676
0
    return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0);
1677
0
  }
1678
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1679
0
    return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0);
1680
0
  }
1681
0
  return 0;
1682
0
}
1683
1684
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
1685
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1686
0
    return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0);
1687
0
  }
1688
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1689
0
    return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0);
1690
0
  }
1691
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1692
0
    return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0);
1693
0
  }
1694
0
  return 0;
1695
0
}
1696
1697
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) {
1698
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1699
0
    return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0);
1700
0
  }
1701
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1702
0
    return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0);
1703
0
  }
1704
0
  return 0;
1705
0
}
1706
1707
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) {
1708
0
  if ((Subtarget->hasAVX512())) {
1709
0
    return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0);
1710
0
  }
1711
0
  return 0;
1712
0
}
1713
1714
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1715
0
switch (RetVT.SimpleTy) {
1716
0
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
1717
0
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
1718
0
  case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
1719
0
  case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
1720
0
  case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
1721
0
  case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
1722
0
  default: return 0;
1723
0
}
1724
0
}
1725
1726
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
1727
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1728
0
    return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0);
1729
0
  }
1730
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1731
0
    return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0);
1732
0
  }
1733
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1734
0
    return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0);
1735
0
  }
1736
0
  return 0;
1737
0
}
1738
1739
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
1740
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1741
0
    return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0);
1742
0
  }
1743
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1744
0
    return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0);
1745
0
  }
1746
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1747
0
    return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0);
1748
0
  }
1749
0
  return 0;
1750
0
}
1751
1752
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) {
1753
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1754
0
    return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0);
1755
0
  }
1756
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1757
0
    return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0);
1758
0
  }
1759
0
  return 0;
1760
0
}
1761
1762
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1763
0
switch (RetVT.SimpleTy) {
1764
0
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
1765
0
  case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
1766
0
  case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
1767
0
  default: return 0;
1768
0
}
1769
0
}
1770
1771
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1772
0
  if (RetVT.SimpleTy != MVT::v2i64)
1773
0
    return 0;
1774
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1775
0
    return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0);
1776
0
  }
1777
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1778
0
    return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0);
1779
0
  }
1780
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1781
0
    return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0);
1782
0
  }
1783
0
  return 0;
1784
0
}
1785
1786
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) {
1787
0
  switch (VT.SimpleTy) {
1788
0
  case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
1789
0
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
1790
0
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
1791
0
  default: return 0;
1792
0
  }
1793
0
}
1794
1795
// FastEmit functions for ISD::SINT_TO_FP.
1796
1797
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
1798
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1799
0
    return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0);
1800
0
  }
1801
0
  return 0;
1802
0
}
1803
1804
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
1805
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1806
0
    return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0);
1807
0
  }
1808
0
  return 0;
1809
0
}
1810
1811
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
1812
0
switch (RetVT.SimpleTy) {
1813
0
  case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
1814
0
  case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
1815
0
  default: return 0;
1816
0
}
1817
0
}
1818
1819
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
1820
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1821
0
    return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0);
1822
0
  }
1823
0
  return 0;
1824
0
}
1825
1826
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
1827
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1828
0
    return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0);
1829
0
  }
1830
0
  return 0;
1831
0
}
1832
1833
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
1834
0
switch (RetVT.SimpleTy) {
1835
0
  case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
1836
0
  case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
1837
0
  default: return 0;
1838
0
}
1839
0
}
1840
1841
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1842
0
  if (RetVT.SimpleTy != MVT::v8f16)
1843
0
    return 0;
1844
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1845
0
    return fastEmitInst_r(X86::VCVTW2PHZ128rr, &X86::VR128XRegClass, Op0);
1846
0
  }
1847
0
  return 0;
1848
0
}
1849
1850
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
1851
0
  if (RetVT.SimpleTy != MVT::v16f16)
1852
0
    return 0;
1853
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1854
0
    return fastEmitInst_r(X86::VCVTW2PHZ256rr, &X86::VR256XRegClass, Op0);
1855
0
  }
1856
0
  return 0;
1857
0
}
1858
1859
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
1860
0
  if (RetVT.SimpleTy != MVT::v32f16)
1861
0
    return 0;
1862
0
  if ((Subtarget->hasFP16())) {
1863
0
    return fastEmitInst_r(X86::VCVTW2PHZrr, &X86::VR512RegClass, Op0);
1864
0
  }
1865
0
  return 0;
1866
0
}
1867
1868
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
1869
0
  if ((Subtarget->hasVLX())) {
1870
0
    return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0);
1871
0
  }
1872
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1873
0
    return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0);
1874
0
  }
1875
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1876
0
    return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0);
1877
0
  }
1878
0
  return 0;
1879
0
}
1880
1881
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
1882
0
  if ((Subtarget->hasVLX())) {
1883
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0);
1884
0
  }
1885
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1886
0
    return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0);
1887
0
  }
1888
0
  return 0;
1889
0
}
1890
1891
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1892
0
switch (RetVT.SimpleTy) {
1893
0
  case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
1894
0
  case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
1895
0
  default: return 0;
1896
0
}
1897
0
}
1898
1899
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
1900
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
1901
0
    return fastEmitInst_r(X86::VCVTDQ2PHZ256rr, &X86::VR128XRegClass, Op0);
1902
0
  }
1903
0
  return 0;
1904
0
}
1905
1906
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
1907
0
  if ((Subtarget->hasVLX())) {
1908
0
    return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0);
1909
0
  }
1910
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1911
0
    return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0);
1912
0
  }
1913
0
  return 0;
1914
0
}
1915
1916
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
1917
0
  if ((Subtarget->hasAVX512())) {
1918
0
    return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0);
1919
0
  }
1920
0
  return 0;
1921
0
}
1922
1923
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
1924
0
switch (RetVT.SimpleTy) {
1925
0
  case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
1926
0
  case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
1927
0
  case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
1928
0
  default: return 0;
1929
0
}
1930
0
}
1931
1932
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
1933
0
  if ((Subtarget->hasFP16())) {
1934
0
    return fastEmitInst_r(X86::VCVTDQ2PHZrr, &X86::VR256XRegClass, Op0);
1935
0
  }
1936
0
  return 0;
1937
0
}
1938
1939
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
1940
0
  if ((Subtarget->hasAVX512())) {
1941
0
    return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0);
1942
0
  }
1943
0
  return 0;
1944
0
}
1945
1946
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
1947
0
switch (RetVT.SimpleTy) {
1948
0
  case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
1949
0
  case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
1950
0
  default: return 0;
1951
0
}
1952
0
}
1953
1954
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
1955
0
  if (RetVT.SimpleTy != MVT::v2f64)
1956
0
    return 0;
1957
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1958
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0);
1959
0
  }
1960
0
  return 0;
1961
0
}
1962
1963
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
1964
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1965
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0);
1966
0
  }
1967
0
  return 0;
1968
0
}
1969
1970
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
1971
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1972
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0);
1973
0
  }
1974
0
  return 0;
1975
0
}
1976
1977
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
1978
0
switch (RetVT.SimpleTy) {
1979
0
  case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
1980
0
  case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
1981
0
  default: return 0;
1982
0
}
1983
0
}
1984
1985
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
1986
0
  if ((Subtarget->hasFP16())) {
1987
0
    return fastEmitInst_r(X86::VCVTQQ2PHZrr, &X86::VR128XRegClass, Op0);
1988
0
  }
1989
0
  return 0;
1990
0
}
1991
1992
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
1993
0
  if ((Subtarget->hasDQI())) {
1994
0
    return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0);
1995
0
  }
1996
0
  return 0;
1997
0
}
1998
1999
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
2000
0
  if ((Subtarget->hasDQI())) {
2001
0
    return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0);
2002
0
  }
2003
0
  return 0;
2004
0
}
2005
2006
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
2007
0
switch (RetVT.SimpleTy) {
2008
0
  case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2009
0
  case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2010
0
  case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2011
0
  default: return 0;
2012
0
}
2013
0
}
2014
2015
0
unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2016
0
  switch (VT.SimpleTy) {
2017
0
  case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2018
0
  case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2019
0
  case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2020
0
  case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2021
0
  case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2022
0
  case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2023
0
  case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2024
0
  case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2025
0
  case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2026
0
  case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2027
0
  case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2028
0
  default: return 0;
2029
0
  }
2030
0
}
2031
2032
// FastEmit functions for ISD::STRICT_FP_EXTEND.
2033
2034
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
2035
0
  if (RetVT.SimpleTy != MVT::f64)
2036
0
    return 0;
2037
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2038
0
    return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0);
2039
0
  }
2040
0
  return 0;
2041
0
}
2042
2043
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) {
2044
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2045
0
    return fastEmitInst_r(X86::VCVTPH2PSXZ256rr, &X86::VR256XRegClass, Op0);
2046
0
  }
2047
0
  return 0;
2048
0
}
2049
2050
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) {
2051
0
  if ((Subtarget->hasFP16())) {
2052
0
    return fastEmitInst_r(X86::VCVTPH2PDZrr, &X86::VR512RegClass, Op0);
2053
0
  }
2054
0
  return 0;
2055
0
}
2056
2057
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2058
0
switch (RetVT.SimpleTy) {
2059
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0);
2060
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0);
2061
0
  default: return 0;
2062
0
}
2063
0
}
2064
2065
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
2066
0
  if (RetVT.SimpleTy != MVT::v16f32)
2067
0
    return 0;
2068
0
  if ((Subtarget->hasFP16())) {
2069
0
    return fastEmitInst_r(X86::VCVTPH2PSXZrr, &X86::VR512RegClass, Op0);
2070
0
  }
2071
0
  return 0;
2072
0
}
2073
2074
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2075
0
  if (RetVT.SimpleTy != MVT::v4f64)
2076
0
    return 0;
2077
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2078
0
    return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0);
2079
0
  }
2080
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2081
0
    return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0);
2082
0
  }
2083
0
  return 0;
2084
0
}
2085
2086
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
2087
0
  if (RetVT.SimpleTy != MVT::v8f64)
2088
0
    return 0;
2089
0
  if ((Subtarget->hasAVX512())) {
2090
0
    return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0);
2091
0
  }
2092
0
  return 0;
2093
0
}
2094
2095
0
unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
2096
0
  switch (VT.SimpleTy) {
2097
0
  case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2098
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0);
2099
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0);
2100
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0);
2101
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0);
2102
0
  default: return 0;
2103
0
  }
2104
0
}
2105
2106
// FastEmit functions for ISD::STRICT_FP_ROUND.
2107
2108
0
unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
2109
0
  if (RetVT.SimpleTy != MVT::f32)
2110
0
    return 0;
2111
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2112
0
    return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0);
2113
0
  }
2114
0
  return 0;
2115
0
}
2116
2117
0
unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
2118
0
  switch (VT.SimpleTy) {
2119
0
  case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
2120
0
  default: return 0;
2121
0
  }
2122
0
}
2123
2124
// FastEmit functions for ISD::STRICT_FP_TO_SINT.
2125
2126
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
2127
0
  if ((Subtarget->hasFP16())) {
2128
0
    return fastEmitInst_r(X86::VCVTTSH2SIZrr, &X86::GR32RegClass, Op0);
2129
0
  }
2130
0
  return 0;
2131
0
}
2132
2133
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
2134
0
  if ((Subtarget->hasFP16())) {
2135
0
    return fastEmitInst_r(X86::VCVTTSH2SI64Zrr, &X86::GR64RegClass, Op0);
2136
0
  }
2137
0
  return 0;
2138
0
}
2139
2140
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2141
0
switch (RetVT.SimpleTy) {
2142
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
2143
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
2144
0
  default: return 0;
2145
0
}
2146
0
}
2147
2148
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
2149
0
  if ((Subtarget->hasAVX512())) {
2150
0
    return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0);
2151
0
  }
2152
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2153
0
    return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0);
2154
0
  }
2155
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2156
0
    return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0);
2157
0
  }
2158
0
  return 0;
2159
0
}
2160
2161
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
2162
0
  if ((Subtarget->hasAVX512())) {
2163
0
    return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0);
2164
0
  }
2165
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2166
0
    return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0);
2167
0
  }
2168
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2169
0
    return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0);
2170
0
  }
2171
0
  return 0;
2172
0
}
2173
2174
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2175
0
switch (RetVT.SimpleTy) {
2176
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
2177
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
2178
0
  default: return 0;
2179
0
}
2180
0
}
2181
2182
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
2183
0
  if ((Subtarget->hasAVX512())) {
2184
0
    return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0);
2185
0
  }
2186
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2187
0
    return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0);
2188
0
  }
2189
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2190
0
    return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0);
2191
0
  }
2192
0
  return 0;
2193
0
}
2194
2195
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
2196
0
  if ((Subtarget->hasAVX512())) {
2197
0
    return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0);
2198
0
  }
2199
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2200
0
    return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0);
2201
0
  }
2202
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2203
0
    return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0);
2204
0
  }
2205
0
  return 0;
2206
0
}
2207
2208
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2209
0
switch (RetVT.SimpleTy) {
2210
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
2211
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
2212
0
  default: return 0;
2213
0
}
2214
0
}
2215
2216
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
2217
0
  if (RetVT.SimpleTy != MVT::v4i32)
2218
0
    return 0;
2219
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2220
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0);
2221
0
  }
2222
0
  return 0;
2223
0
}
2224
2225
0
unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2226
0
  switch (VT.SimpleTy) {
2227
0
  case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
2228
0
  case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
2229
0
  case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
2230
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0);
2231
0
  default: return 0;
2232
0
  }
2233
0
}
2234
2235
// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2236
2237
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
2238
0
  if ((Subtarget->hasFP16())) {
2239
0
    return fastEmitInst_r(X86::VCVTTSH2USIZrr, &X86::GR32RegClass, Op0);
2240
0
  }
2241
0
  return 0;
2242
0
}
2243
2244
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
2245
0
  if ((Subtarget->hasFP16())) {
2246
0
    return fastEmitInst_r(X86::VCVTTSH2USI64Zrr, &X86::GR64RegClass, Op0);
2247
0
  }
2248
0
  return 0;
2249
0
}
2250
2251
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2252
0
switch (RetVT.SimpleTy) {
2253
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
2254
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
2255
0
  default: return 0;
2256
0
}
2257
0
}
2258
2259
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
2260
0
  if ((Subtarget->hasAVX512())) {
2261
0
    return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0);
2262
0
  }
2263
0
  return 0;
2264
0
}
2265
2266
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
2267
0
  if ((Subtarget->hasAVX512())) {
2268
0
    return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0);
2269
0
  }
2270
0
  return 0;
2271
0
}
2272
2273
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2274
0
switch (RetVT.SimpleTy) {
2275
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
2276
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
2277
0
  default: return 0;
2278
0
}
2279
0
}
2280
2281
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
2282
0
  if ((Subtarget->hasAVX512())) {
2283
0
    return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0);
2284
0
  }
2285
0
  return 0;
2286
0
}
2287
2288
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
2289
0
  if ((Subtarget->hasAVX512())) {
2290
0
    return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0);
2291
0
  }
2292
0
  return 0;
2293
0
}
2294
2295
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2296
0
switch (RetVT.SimpleTy) {
2297
0
  case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
2298
0
  case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
2299
0
  default: return 0;
2300
0
}
2301
0
}
2302
2303
0
unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2304
0
  switch (VT.SimpleTy) {
2305
0
  case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
2306
0
  case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
2307
0
  case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
2308
0
  default: return 0;
2309
0
  }
2310
0
}
2311
2312
// FastEmit functions for ISD::STRICT_FSQRT.
2313
2314
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2315
0
  if (RetVT.SimpleTy != MVT::f32)
2316
0
    return 0;
2317
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2318
0
    return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0);
2319
0
  }
2320
0
  if ((!Subtarget->hasSSE1())) {
2321
0
    return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0);
2322
0
  }
2323
0
  return 0;
2324
0
}
2325
2326
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2327
0
  if (RetVT.SimpleTy != MVT::f64)
2328
0
    return 0;
2329
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2330
0
    return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0);
2331
0
  }
2332
0
  if ((!Subtarget->hasSSE2())) {
2333
0
    return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0);
2334
0
  }
2335
0
  return 0;
2336
0
}
2337
2338
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) {
2339
0
  if (RetVT.SimpleTy != MVT::f80)
2340
0
    return 0;
2341
0
  if ((Subtarget->hasX87())) {
2342
0
    return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0);
2343
0
  }
2344
0
  return 0;
2345
0
}
2346
2347
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2348
0
  if (RetVT.SimpleTy != MVT::v8f16)
2349
0
    return 0;
2350
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2351
0
    return fastEmitInst_r(X86::VSQRTPHZ128r, &X86::VR128XRegClass, Op0);
2352
0
  }
2353
0
  return 0;
2354
0
}
2355
2356
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
2357
0
  if (RetVT.SimpleTy != MVT::v16f16)
2358
0
    return 0;
2359
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2360
0
    return fastEmitInst_r(X86::VSQRTPHZ256r, &X86::VR256XRegClass, Op0);
2361
0
  }
2362
0
  return 0;
2363
0
}
2364
2365
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
2366
0
  if (RetVT.SimpleTy != MVT::v32f16)
2367
0
    return 0;
2368
0
  if ((Subtarget->hasFP16())) {
2369
0
    return fastEmitInst_r(X86::VSQRTPHZr, &X86::VR512RegClass, Op0);
2370
0
  }
2371
0
  return 0;
2372
0
}
2373
2374
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2375
0
  if (RetVT.SimpleTy != MVT::v4f32)
2376
0
    return 0;
2377
0
  if ((Subtarget->hasVLX())) {
2378
0
    return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0);
2379
0
  }
2380
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2381
0
    return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0);
2382
0
  }
2383
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2384
0
    return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0);
2385
0
  }
2386
0
  return 0;
2387
0
}
2388
2389
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
2390
0
  if (RetVT.SimpleTy != MVT::v8f32)
2391
0
    return 0;
2392
0
  if ((Subtarget->hasVLX())) {
2393
0
    return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0);
2394
0
  }
2395
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2396
0
    return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0);
2397
0
  }
2398
0
  return 0;
2399
0
}
2400
2401
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
2402
0
  if (RetVT.SimpleTy != MVT::v16f32)
2403
0
    return 0;
2404
0
  if ((Subtarget->hasAVX512())) {
2405
0
    return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0);
2406
0
  }
2407
0
  return 0;
2408
0
}
2409
2410
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
2411
0
  if (RetVT.SimpleTy != MVT::v2f64)
2412
0
    return 0;
2413
0
  if ((Subtarget->hasVLX())) {
2414
0
    return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0);
2415
0
  }
2416
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2417
0
    return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0);
2418
0
  }
2419
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2420
0
    return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0);
2421
0
  }
2422
0
  return 0;
2423
0
}
2424
2425
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
2426
0
  if (RetVT.SimpleTy != MVT::v4f64)
2427
0
    return 0;
2428
0
  if ((Subtarget->hasVLX())) {
2429
0
    return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0);
2430
0
  }
2431
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2432
0
    return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0);
2433
0
  }
2434
0
  return 0;
2435
0
}
2436
2437
0
unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
2438
0
  if (RetVT.SimpleTy != MVT::v8f64)
2439
0
    return 0;
2440
0
  if ((Subtarget->hasAVX512())) {
2441
0
    return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0);
2442
0
  }
2443
0
  return 0;
2444
0
}
2445
2446
0
unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
2447
0
  switch (VT.SimpleTy) {
2448
0
  case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
2449
0
  case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
2450
0
  case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0);
2451
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
2452
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0);
2453
0
  case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0);
2454
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
2455
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0);
2456
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0);
2457
0
  case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
2458
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0);
2459
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0);
2460
0
  default: return 0;
2461
0
  }
2462
0
}
2463
2464
// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2465
2466
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
2467
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2468
0
    return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0);
2469
0
  }
2470
0
  return 0;
2471
0
}
2472
2473
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
2474
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2475
0
    return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0);
2476
0
  }
2477
0
  return 0;
2478
0
}
2479
2480
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
2481
0
switch (RetVT.SimpleTy) {
2482
0
  case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
2483
0
  case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
2484
0
  default: return 0;
2485
0
}
2486
0
}
2487
2488
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
2489
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2490
0
    return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0);
2491
0
  }
2492
0
  return 0;
2493
0
}
2494
2495
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
2496
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2497
0
    return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0);
2498
0
  }
2499
0
  return 0;
2500
0
}
2501
2502
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
2503
0
switch (RetVT.SimpleTy) {
2504
0
  case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
2505
0
  case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
2506
0
  default: return 0;
2507
0
}
2508
0
}
2509
2510
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2511
0
  if (RetVT.SimpleTy != MVT::v8f16)
2512
0
    return 0;
2513
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2514
0
    return fastEmitInst_r(X86::VCVTW2PHZ128rr, &X86::VR128XRegClass, Op0);
2515
0
  }
2516
0
  return 0;
2517
0
}
2518
2519
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2520
0
  if (RetVT.SimpleTy != MVT::v16f16)
2521
0
    return 0;
2522
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2523
0
    return fastEmitInst_r(X86::VCVTW2PHZ256rr, &X86::VR256XRegClass, Op0);
2524
0
  }
2525
0
  return 0;
2526
0
}
2527
2528
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2529
0
  if (RetVT.SimpleTy != MVT::v32f16)
2530
0
    return 0;
2531
0
  if ((Subtarget->hasFP16())) {
2532
0
    return fastEmitInst_r(X86::VCVTW2PHZrr, &X86::VR512RegClass, Op0);
2533
0
  }
2534
0
  return 0;
2535
0
}
2536
2537
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
2538
0
  if ((Subtarget->hasVLX())) {
2539
0
    return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0);
2540
0
  }
2541
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2542
0
    return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0);
2543
0
  }
2544
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2545
0
    return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0);
2546
0
  }
2547
0
  return 0;
2548
0
}
2549
2550
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
2551
0
  if ((Subtarget->hasVLX())) {
2552
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0);
2553
0
  }
2554
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2555
0
    return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0);
2556
0
  }
2557
0
  return 0;
2558
0
}
2559
2560
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2561
0
switch (RetVT.SimpleTy) {
2562
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2563
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2564
0
  default: return 0;
2565
0
}
2566
0
}
2567
2568
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
2569
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2570
0
    return fastEmitInst_r(X86::VCVTDQ2PHZ256rr, &X86::VR128XRegClass, Op0);
2571
0
  }
2572
0
  return 0;
2573
0
}
2574
2575
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
2576
0
  if ((Subtarget->hasVLX())) {
2577
0
    return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0);
2578
0
  }
2579
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2580
0
    return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0);
2581
0
  }
2582
0
  return 0;
2583
0
}
2584
2585
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
2586
0
  if ((Subtarget->hasAVX512())) {
2587
0
    return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0);
2588
0
  }
2589
0
  return 0;
2590
0
}
2591
2592
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2593
0
switch (RetVT.SimpleTy) {
2594
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2595
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2596
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2597
0
  default: return 0;
2598
0
}
2599
0
}
2600
2601
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
2602
0
  if ((Subtarget->hasFP16())) {
2603
0
    return fastEmitInst_r(X86::VCVTDQ2PHZrr, &X86::VR256XRegClass, Op0);
2604
0
  }
2605
0
  return 0;
2606
0
}
2607
2608
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
2609
0
  if ((Subtarget->hasAVX512())) {
2610
0
    return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0);
2611
0
  }
2612
0
  return 0;
2613
0
}
2614
2615
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2616
0
switch (RetVT.SimpleTy) {
2617
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2618
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2619
0
  default: return 0;
2620
0
}
2621
0
}
2622
2623
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2624
0
  if (RetVT.SimpleTy != MVT::v2f64)
2625
0
    return 0;
2626
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2627
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0);
2628
0
  }
2629
0
  return 0;
2630
0
}
2631
2632
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
2633
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2634
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0);
2635
0
  }
2636
0
  return 0;
2637
0
}
2638
2639
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
2640
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2641
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0);
2642
0
  }
2643
0
  return 0;
2644
0
}
2645
2646
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
2647
0
switch (RetVT.SimpleTy) {
2648
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2649
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2650
0
  default: return 0;
2651
0
}
2652
0
}
2653
2654
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
2655
0
  if ((Subtarget->hasFP16())) {
2656
0
    return fastEmitInst_r(X86::VCVTQQ2PHZrr, &X86::VR128XRegClass, Op0);
2657
0
  }
2658
0
  return 0;
2659
0
}
2660
2661
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
2662
0
  if ((Subtarget->hasDQI())) {
2663
0
    return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0);
2664
0
  }
2665
0
  return 0;
2666
0
}
2667
2668
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
2669
0
  if ((Subtarget->hasDQI())) {
2670
0
    return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0);
2671
0
  }
2672
0
  return 0;
2673
0
}
2674
2675
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
2676
0
switch (RetVT.SimpleTy) {
2677
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2678
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2679
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2680
0
  default: return 0;
2681
0
}
2682
0
}
2683
2684
0
unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2685
0
  switch (VT.SimpleTy) {
2686
0
  case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
2687
0
  case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
2688
0
  case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2689
0
  case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2690
0
  case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2691
0
  case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2692
0
  case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2693
0
  case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2694
0
  case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2695
0
  case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2696
0
  case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2697
0
  default: return 0;
2698
0
  }
2699
0
}
2700
2701
// FastEmit functions for ISD::STRICT_UINT_TO_FP.
2702
2703
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2704
0
  if (RetVT.SimpleTy != MVT::v8f16)
2705
0
    return 0;
2706
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2707
0
    return fastEmitInst_r(X86::VCVTUW2PHZ128rr, &X86::VR128XRegClass, Op0);
2708
0
  }
2709
0
  return 0;
2710
0
}
2711
2712
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2713
0
  if (RetVT.SimpleTy != MVT::v16f16)
2714
0
    return 0;
2715
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2716
0
    return fastEmitInst_r(X86::VCVTUW2PHZ256rr, &X86::VR256XRegClass, Op0);
2717
0
  }
2718
0
  return 0;
2719
0
}
2720
2721
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2722
0
  if (RetVT.SimpleTy != MVT::v32f16)
2723
0
    return 0;
2724
0
  if ((Subtarget->hasFP16())) {
2725
0
    return fastEmitInst_r(X86::VCVTUW2PHZrr, &X86::VR512RegClass, Op0);
2726
0
  }
2727
0
  return 0;
2728
0
}
2729
2730
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
2731
0
  if ((Subtarget->hasVLX())) {
2732
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0);
2733
0
  }
2734
0
  return 0;
2735
0
}
2736
2737
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
2738
0
  if ((Subtarget->hasVLX())) {
2739
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0);
2740
0
  }
2741
0
  return 0;
2742
0
}
2743
2744
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2745
0
switch (RetVT.SimpleTy) {
2746
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
2747
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
2748
0
  default: return 0;
2749
0
}
2750
0
}
2751
2752
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
2753
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
2754
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZ256rr, &X86::VR128XRegClass, Op0);
2755
0
  }
2756
0
  return 0;
2757
0
}
2758
2759
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
2760
0
  if ((Subtarget->hasVLX())) {
2761
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0);
2762
0
  }
2763
0
  return 0;
2764
0
}
2765
2766
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
2767
0
  if ((Subtarget->hasAVX512())) {
2768
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0);
2769
0
  }
2770
0
  return 0;
2771
0
}
2772
2773
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2774
0
switch (RetVT.SimpleTy) {
2775
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
2776
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
2777
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
2778
0
  default: return 0;
2779
0
}
2780
0
}
2781
2782
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
2783
0
  if ((Subtarget->hasFP16())) {
2784
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZrr, &X86::VR256XRegClass, Op0);
2785
0
  }
2786
0
  return 0;
2787
0
}
2788
2789
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
2790
0
  if ((Subtarget->hasAVX512())) {
2791
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0);
2792
0
  }
2793
0
  return 0;
2794
0
}
2795
2796
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2797
0
switch (RetVT.SimpleTy) {
2798
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
2799
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
2800
0
  default: return 0;
2801
0
}
2802
0
}
2803
2804
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2805
0
  if (RetVT.SimpleTy != MVT::v2f64)
2806
0
    return 0;
2807
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2808
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0);
2809
0
  }
2810
0
  return 0;
2811
0
}
2812
2813
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
2814
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2815
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0);
2816
0
  }
2817
0
  return 0;
2818
0
}
2819
2820
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
2821
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2822
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0);
2823
0
  }
2824
0
  return 0;
2825
0
}
2826
2827
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
2828
0
switch (RetVT.SimpleTy) {
2829
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
2830
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
2831
0
  default: return 0;
2832
0
}
2833
0
}
2834
2835
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
2836
0
  if ((Subtarget->hasFP16())) {
2837
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZrr, &X86::VR128XRegClass, Op0);
2838
0
  }
2839
0
  return 0;
2840
0
}
2841
2842
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
2843
0
  if ((Subtarget->hasDQI())) {
2844
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0);
2845
0
  }
2846
0
  return 0;
2847
0
}
2848
2849
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
2850
0
  if ((Subtarget->hasDQI())) {
2851
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0);
2852
0
  }
2853
0
  return 0;
2854
0
}
2855
2856
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
2857
0
switch (RetVT.SimpleTy) {
2858
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
2859
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
2860
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
2861
0
  default: return 0;
2862
0
}
2863
0
}
2864
2865
0
unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2866
0
  switch (VT.SimpleTy) {
2867
0
  case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2868
0
  case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
2869
0
  case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
2870
0
  case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2871
0
  case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
2872
0
  case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
2873
0
  case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
2874
0
  case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
2875
0
  case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
2876
0
  default: return 0;
2877
0
  }
2878
0
}
2879
2880
// FastEmit functions for ISD::TRUNCATE.
2881
2882
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0) {
2883
0
  if (RetVT.SimpleTy != MVT::i8)
2884
0
    return 0;
2885
0
  if ((Subtarget->is64Bit())) {
2886
0
    return fastEmitInst_extractsubreg(RetVT, Op0, X86::sub_8bit);
2887
0
  }
2888
0
  return 0;
2889
0
}
2890
2891
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0) {
2892
0
  if ((Subtarget->is64Bit())) {
2893
0
    return fastEmitInst_extractsubreg(MVT::i8, Op0, X86::sub_8bit);
2894
0
  }
2895
0
  return 0;
2896
0
}
2897
2898
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0) {
2899
0
  return fastEmitInst_extractsubreg(MVT::i16, Op0, X86::sub_16bit);
2900
0
}
2901
2902
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0) {
2903
0
switch (RetVT.SimpleTy) {
2904
0
  case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0);
2905
0
  case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0);
2906
0
  default: return 0;
2907
0
}
2908
0
}
2909
2910
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0) {
2911
0
  return fastEmitInst_extractsubreg(MVT::i8, Op0, X86::sub_8bit);
2912
0
}
2913
2914
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0) {
2915
0
  return fastEmitInst_extractsubreg(MVT::i16, Op0, X86::sub_16bit);
2916
0
}
2917
2918
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0) {
2919
0
  return fastEmitInst_extractsubreg(MVT::i32, Op0, X86::sub_32bit);
2920
0
}
2921
2922
0
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0) {
2923
0
switch (RetVT.SimpleTy) {
2924
0
  case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0);
2925
0
  case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0);
2926
0
  case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0);
2927
0
  default: return 0;
2928
0
}
2929
0
}
2930
2931
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
2932
0
  if (RetVT.SimpleTy != MVT::v16i8)
2933
0
    return 0;
2934
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2935
0
    return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0);
2936
0
  }
2937
0
  return 0;
2938
0
}
2939
2940
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
2941
0
  if (RetVT.SimpleTy != MVT::v32i8)
2942
0
    return 0;
2943
0
  if ((Subtarget->hasBWI())) {
2944
0
    return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0);
2945
0
  }
2946
0
  return 0;
2947
0
}
2948
2949
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
2950
0
  if (RetVT.SimpleTy != MVT::v8i16)
2951
0
    return 0;
2952
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2953
0
    return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0);
2954
0
  }
2955
0
  return 0;
2956
0
}
2957
2958
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
2959
0
  if ((Subtarget->hasAVX512())) {
2960
0
    return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0);
2961
0
  }
2962
0
  return 0;
2963
0
}
2964
2965
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
2966
0
  if ((Subtarget->hasAVX512())) {
2967
0
    return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0);
2968
0
  }
2969
0
  return 0;
2970
0
}
2971
2972
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
2973
0
switch (RetVT.SimpleTy) {
2974
0
  case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0);
2975
0
  case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0);
2976
0
  default: return 0;
2977
0
}
2978
0
}
2979
2980
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
2981
0
  if (RetVT.SimpleTy != MVT::v4i32)
2982
0
    return 0;
2983
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2984
0
    return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0);
2985
0
  }
2986
0
  return 0;
2987
0
}
2988
2989
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
2990
0
  if ((Subtarget->hasAVX512())) {
2991
0
    return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0);
2992
0
  }
2993
0
  return 0;
2994
0
}
2995
2996
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
2997
0
  if ((Subtarget->hasAVX512())) {
2998
0
    return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0);
2999
0
  }
3000
0
  return 0;
3001
0
}
3002
3003
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3004
0
switch (RetVT.SimpleTy) {
3005
0
  case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0);
3006
0
  case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0);
3007
0
  default: return 0;
3008
0
}
3009
0
}
3010
3011
0
unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) {
3012
0
  switch (VT.SimpleTy) {
3013
0
  case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0);
3014
0
  case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0);
3015
0
  case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
3016
0
  case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0);
3017
0
  case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0);
3018
0
  case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0);
3019
0
  case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0);
3020
0
  case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0);
3021
0
  case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0);
3022
0
  default: return 0;
3023
0
  }
3024
0
}
3025
3026
// FastEmit functions for ISD::UINT_TO_FP.
3027
3028
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3029
0
  if (RetVT.SimpleTy != MVT::v8f16)
3030
0
    return 0;
3031
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3032
0
    return fastEmitInst_r(X86::VCVTUW2PHZ128rr, &X86::VR128XRegClass, Op0);
3033
0
  }
3034
0
  return 0;
3035
0
}
3036
3037
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
3038
0
  if (RetVT.SimpleTy != MVT::v16f16)
3039
0
    return 0;
3040
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3041
0
    return fastEmitInst_r(X86::VCVTUW2PHZ256rr, &X86::VR256XRegClass, Op0);
3042
0
  }
3043
0
  return 0;
3044
0
}
3045
3046
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
3047
0
  if (RetVT.SimpleTy != MVT::v32f16)
3048
0
    return 0;
3049
0
  if ((Subtarget->hasFP16())) {
3050
0
    return fastEmitInst_r(X86::VCVTUW2PHZrr, &X86::VR512RegClass, Op0);
3051
0
  }
3052
0
  return 0;
3053
0
}
3054
3055
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) {
3056
0
  if ((Subtarget->hasVLX())) {
3057
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0);
3058
0
  }
3059
0
  return 0;
3060
0
}
3061
3062
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) {
3063
0
  if ((Subtarget->hasVLX())) {
3064
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0);
3065
0
  }
3066
0
  return 0;
3067
0
}
3068
3069
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3070
0
switch (RetVT.SimpleTy) {
3071
0
  case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0);
3072
0
  case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0);
3073
0
  default: return 0;
3074
0
}
3075
0
}
3076
3077
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) {
3078
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3079
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZ256rr, &X86::VR128XRegClass, Op0);
3080
0
  }
3081
0
  return 0;
3082
0
}
3083
3084
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) {
3085
0
  if ((Subtarget->hasVLX())) {
3086
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0);
3087
0
  }
3088
0
  return 0;
3089
0
}
3090
3091
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) {
3092
0
  if ((Subtarget->hasAVX512())) {
3093
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0);
3094
0
  }
3095
0
  return 0;
3096
0
}
3097
3098
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3099
0
switch (RetVT.SimpleTy) {
3100
0
  case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0);
3101
0
  case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0);
3102
0
  case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0);
3103
0
  default: return 0;
3104
0
}
3105
0
}
3106
3107
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) {
3108
0
  if ((Subtarget->hasFP16())) {
3109
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZrr, &X86::VR256XRegClass, Op0);
3110
0
  }
3111
0
  return 0;
3112
0
}
3113
3114
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) {
3115
0
  if ((Subtarget->hasAVX512())) {
3116
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0);
3117
0
  }
3118
0
  return 0;
3119
0
}
3120
3121
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
3122
0
switch (RetVT.SimpleTy) {
3123
0
  case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0);
3124
0
  case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0);
3125
0
  default: return 0;
3126
0
}
3127
0
}
3128
3129
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
3130
0
  if (RetVT.SimpleTy != MVT::v2f64)
3131
0
    return 0;
3132
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3133
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0);
3134
0
  }
3135
0
  return 0;
3136
0
}
3137
3138
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) {
3139
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3140
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0);
3141
0
  }
3142
0
  return 0;
3143
0
}
3144
3145
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) {
3146
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3147
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0);
3148
0
  }
3149
0
  return 0;
3150
0
}
3151
3152
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3153
0
switch (RetVT.SimpleTy) {
3154
0
  case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0);
3155
0
  case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0);
3156
0
  default: return 0;
3157
0
}
3158
0
}
3159
3160
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) {
3161
0
  if ((Subtarget->hasFP16())) {
3162
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZrr, &X86::VR128XRegClass, Op0);
3163
0
  }
3164
0
  return 0;
3165
0
}
3166
3167
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) {
3168
0
  if ((Subtarget->hasDQI())) {
3169
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0);
3170
0
  }
3171
0
  return 0;
3172
0
}
3173
3174
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) {
3175
0
  if ((Subtarget->hasDQI())) {
3176
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0);
3177
0
  }
3178
0
  return 0;
3179
0
}
3180
3181
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3182
0
switch (RetVT.SimpleTy) {
3183
0
  case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0);
3184
0
  case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0);
3185
0
  case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0);
3186
0
  default: return 0;
3187
0
}
3188
0
}
3189
3190
0
unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
3191
0
  switch (VT.SimpleTy) {
3192
0
  case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3193
0
  case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0);
3194
0
  case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0);
3195
0
  case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3196
0
  case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0);
3197
0
  case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0);
3198
0
  case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
3199
0
  case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0);
3200
0
  case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0);
3201
0
  default: return 0;
3202
0
  }
3203
0
}
3204
3205
// FastEmit functions for ISD::ZERO_EXTEND.
3206
3207
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) {
3208
0
  if (RetVT.SimpleTy != MVT::i32)
3209
0
    return 0;
3210
0
  return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0);
3211
0
}
3212
3213
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) {
3214
0
  if (RetVT.SimpleTy != MVT::i32)
3215
0
    return 0;
3216
0
  return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0);
3217
0
}
3218
3219
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) {
3220
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3221
0
    return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0);
3222
0
  }
3223
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3224
0
    return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0);
3225
0
  }
3226
0
  return 0;
3227
0
}
3228
3229
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) {
3230
0
  if ((Subtarget->hasAVX512())) {
3231
0
    return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0);
3232
0
  }
3233
0
  return 0;
3234
0
}
3235
3236
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3237
0
switch (RetVT.SimpleTy) {
3238
0
  case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0);
3239
0
  case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0);
3240
0
  default: return 0;
3241
0
}
3242
0
}
3243
3244
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
3245
0
  if (RetVT.SimpleTy != MVT::v32i16)
3246
0
    return 0;
3247
0
  if ((Subtarget->hasBWI())) {
3248
0
    return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0);
3249
0
  }
3250
0
  return 0;
3251
0
}
3252
3253
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) {
3254
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3255
0
    return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0);
3256
0
  }
3257
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3258
0
    return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0);
3259
0
  }
3260
0
  return 0;
3261
0
}
3262
3263
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) {
3264
0
  if ((Subtarget->hasAVX512())) {
3265
0
    return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0);
3266
0
  }
3267
0
  return 0;
3268
0
}
3269
3270
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3271
0
switch (RetVT.SimpleTy) {
3272
0
  case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0);
3273
0
  case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0);
3274
0
  default: return 0;
3275
0
}
3276
0
}
3277
3278
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
3279
0
  if (RetVT.SimpleTy != MVT::v16i32)
3280
0
    return 0;
3281
0
  if ((Subtarget->hasAVX512())) {
3282
0
    return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0);
3283
0
  }
3284
0
  return 0;
3285
0
}
3286
3287
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3288
0
  if (RetVT.SimpleTy != MVT::v4i64)
3289
0
    return 0;
3290
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3291
0
    return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0);
3292
0
  }
3293
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3294
0
    return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0);
3295
0
  }
3296
0
  return 0;
3297
0
}
3298
3299
0
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3300
0
  if (RetVT.SimpleTy != MVT::v8i64)
3301
0
    return 0;
3302
0
  if ((Subtarget->hasAVX512())) {
3303
0
    return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0);
3304
0
  }
3305
0
  return 0;
3306
0
}
3307
3308
0
unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
3309
0
  switch (VT.SimpleTy) {
3310
0
  case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0);
3311
0
  case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0);
3312
0
  case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0);
3313
0
  case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0);
3314
0
  case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0);
3315
0
  case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0);
3316
0
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0);
3317
0
  case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0);
3318
0
  default: return 0;
3319
0
  }
3320
0
}
3321
3322
// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
3323
3324
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
3325
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3326
0
    return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0);
3327
0
  }
3328
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
3329
0
    return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0);
3330
0
  }
3331
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
3332
0
    return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0);
3333
0
  }
3334
0
  return 0;
3335
0
}
3336
3337
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
3338
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3339
0
    return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0);
3340
0
  }
3341
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3342
0
    return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0);
3343
0
  }
3344
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3345
0
    return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0);
3346
0
  }
3347
0
  return 0;
3348
0
}
3349
3350
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) {
3351
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3352
0
    return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0);
3353
0
  }
3354
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3355
0
    return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0);
3356
0
  }
3357
0
  return 0;
3358
0
}
3359
3360
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
3361
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3362
0
    return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0);
3363
0
  }
3364
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3365
0
    return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0);
3366
0
  }
3367
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3368
0
    return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0);
3369
0
  }
3370
0
  return 0;
3371
0
}
3372
3373
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) {
3374
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3375
0
    return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0);
3376
0
  }
3377
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3378
0
    return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0);
3379
0
  }
3380
0
  return 0;
3381
0
}
3382
3383
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) {
3384
0
  if ((Subtarget->hasAVX512())) {
3385
0
    return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0);
3386
0
  }
3387
0
  return 0;
3388
0
}
3389
3390
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3391
0
switch (RetVT.SimpleTy) {
3392
0
  case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0);
3393
0
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0);
3394
0
  case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0);
3395
0
  case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0);
3396
0
  case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0);
3397
0
  case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0);
3398
0
  default: return 0;
3399
0
}
3400
0
}
3401
3402
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
3403
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3404
0
    return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0);
3405
0
  }
3406
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3407
0
    return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0);
3408
0
  }
3409
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3410
0
    return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0);
3411
0
  }
3412
0
  return 0;
3413
0
}
3414
3415
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
3416
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3417
0
    return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0);
3418
0
  }
3419
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3420
0
    return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0);
3421
0
  }
3422
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3423
0
    return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0);
3424
0
  }
3425
0
  return 0;
3426
0
}
3427
3428
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) {
3429
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
3430
0
    return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0);
3431
0
  }
3432
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3433
0
    return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0);
3434
0
  }
3435
0
  return 0;
3436
0
}
3437
3438
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3439
0
switch (RetVT.SimpleTy) {
3440
0
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0);
3441
0
  case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0);
3442
0
  case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0);
3443
0
  default: return 0;
3444
0
}
3445
0
}
3446
3447
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3448
0
  if (RetVT.SimpleTy != MVT::v2i64)
3449
0
    return 0;
3450
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
3451
0
    return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0);
3452
0
  }
3453
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3454
0
    return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0);
3455
0
  }
3456
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3457
0
    return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0);
3458
0
  }
3459
0
  return 0;
3460
0
}
3461
3462
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) {
3463
0
  switch (VT.SimpleTy) {
3464
0
  case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0);
3465
0
  case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0);
3466
0
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0);
3467
0
  default: return 0;
3468
0
  }
3469
0
}
3470
3471
// FastEmit functions for X86ISD::CALL.
3472
3473
0
unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) {
3474
0
  if (RetVT.SimpleTy != MVT::isVoid)
3475
0
    return 0;
3476
0
  if ((!Subtarget->is64Bit())) {
3477
0
    return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0);
3478
0
  }
3479
0
  return 0;
3480
0
}
3481
3482
0
unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
3483
0
  if (RetVT.SimpleTy != MVT::isVoid)
3484
0
    return 0;
3485
0
  if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3486
0
    return fastEmitInst_r(X86::INDIRECT_THUNK_CALL32, &X86::GR32RegClass, Op0);
3487
0
  }
3488
0
  if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3489
0
    return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0);
3490
0
  }
3491
0
  return 0;
3492
0
}
3493
3494
0
unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) {
3495
0
  if (RetVT.SimpleTy != MVT::isVoid)
3496
0
    return 0;
3497
0
  if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3498
0
    return fastEmitInst_r(X86::INDIRECT_THUNK_CALL64, &X86::GR64RegClass, Op0);
3499
0
  }
3500
0
  if ((Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3501
0
    return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0);
3502
0
  }
3503
0
  return 0;
3504
0
}
3505
3506
0
unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
3507
0
  switch (VT.SimpleTy) {
3508
0
  case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0);
3509
0
  case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0);
3510
0
  case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0);
3511
0
  default: return 0;
3512
0
  }
3513
0
}
3514
3515
// FastEmit functions for X86ISD::CONFLICT.
3516
3517
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3518
0
  if (RetVT.SimpleTy != MVT::v4i32)
3519
0
    return 0;
3520
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3521
0
    return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0);
3522
0
  }
3523
0
  return 0;
3524
0
}
3525
3526
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
3527
0
  if (RetVT.SimpleTy != MVT::v8i32)
3528
0
    return 0;
3529
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3530
0
    return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0);
3531
0
  }
3532
0
  return 0;
3533
0
}
3534
3535
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
3536
0
  if (RetVT.SimpleTy != MVT::v16i32)
3537
0
    return 0;
3538
0
  if ((Subtarget->hasCDI())) {
3539
0
    return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0);
3540
0
  }
3541
0
  return 0;
3542
0
}
3543
3544
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
3545
0
  if (RetVT.SimpleTy != MVT::v2i64)
3546
0
    return 0;
3547
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3548
0
    return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0);
3549
0
  }
3550
0
  return 0;
3551
0
}
3552
3553
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
3554
0
  if (RetVT.SimpleTy != MVT::v4i64)
3555
0
    return 0;
3556
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3557
0
    return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0);
3558
0
  }
3559
0
  return 0;
3560
0
}
3561
3562
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
3563
0
  if (RetVT.SimpleTy != MVT::v8i64)
3564
0
    return 0;
3565
0
  if ((Subtarget->hasCDI())) {
3566
0
    return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0);
3567
0
  }
3568
0
  return 0;
3569
0
}
3570
3571
0
unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0) {
3572
0
  switch (VT.SimpleTy) {
3573
0
  case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0);
3574
0
  case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0);
3575
0
  case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0);
3576
0
  case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0);
3577
0
  case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0);
3578
0
  case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0);
3579
0
  default: return 0;
3580
0
  }
3581
0
}
3582
3583
// FastEmit functions for X86ISD::CVTNEPS2BF16.
3584
3585
0
unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3586
0
  if (RetVT.SimpleTy != MVT::v8bf16)
3587
0
    return 0;
3588
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3589
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Z128rr, &X86::VR128XRegClass, Op0);
3590
0
  }
3591
0
  if ((Subtarget->hasAVXNECONVERT())) {
3592
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16rr, &X86::VR128RegClass, Op0);
3593
0
  }
3594
0
  return 0;
3595
0
}
3596
3597
0
unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
3598
0
  if (RetVT.SimpleTy != MVT::v8bf16)
3599
0
    return 0;
3600
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3601
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Z256rr, &X86::VR128XRegClass, Op0);
3602
0
  }
3603
0
  return 0;
3604
0
}
3605
3606
0
unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
3607
0
  if (RetVT.SimpleTy != MVT::v16bf16)
3608
0
    return 0;
3609
0
  if ((Subtarget->hasBF16())) {
3610
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Zrr, &X86::VR256XRegClass, Op0);
3611
0
  }
3612
0
  return 0;
3613
0
}
3614
3615
0
unsigned fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, unsigned Op0) {
3616
0
  switch (VT.SimpleTy) {
3617
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0);
3618
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0);
3619
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0);
3620
0
  default: return 0;
3621
0
  }
3622
0
}
3623
3624
// FastEmit functions for X86ISD::CVTP2SI.
3625
3626
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
3627
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3628
0
    return fastEmitInst_r(X86::VCVTPH2WZ128rr, &X86::VR128XRegClass, Op0);
3629
0
  }
3630
0
  return 0;
3631
0
}
3632
3633
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
3634
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3635
0
    return fastEmitInst_r(X86::VCVTPH2DQZ128rr, &X86::VR128XRegClass, Op0);
3636
0
  }
3637
0
  return 0;
3638
0
}
3639
3640
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
3641
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3642
0
    return fastEmitInst_r(X86::VCVTPH2DQZ256rr, &X86::VR256XRegClass, Op0);
3643
0
  }
3644
0
  return 0;
3645
0
}
3646
3647
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
3648
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3649
0
    return fastEmitInst_r(X86::VCVTPH2QQZ128rr, &X86::VR128XRegClass, Op0);
3650
0
  }
3651
0
  return 0;
3652
0
}
3653
3654
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
3655
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3656
0
    return fastEmitInst_r(X86::VCVTPH2QQZ256rr, &X86::VR256XRegClass, Op0);
3657
0
  }
3658
0
  return 0;
3659
0
}
3660
3661
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
3662
0
  if ((Subtarget->hasFP16())) {
3663
0
    return fastEmitInst_r(X86::VCVTPH2QQZrr, &X86::VR512RegClass, Op0);
3664
0
  }
3665
0
  return 0;
3666
0
}
3667
3668
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3669
0
switch (RetVT.SimpleTy) {
3670
0
  case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
3671
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
3672
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
3673
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
3674
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
3675
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
3676
0
  default: return 0;
3677
0
}
3678
0
}
3679
3680
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
3681
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3682
0
    return fastEmitInst_r(X86::VCVTPH2WZ256rr, &X86::VR256XRegClass, Op0);
3683
0
  }
3684
0
  return 0;
3685
0
}
3686
3687
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
3688
0
  if ((Subtarget->hasFP16())) {
3689
0
    return fastEmitInst_r(X86::VCVTPH2DQZrr, &X86::VR512RegClass, Op0);
3690
0
  }
3691
0
  return 0;
3692
0
}
3693
3694
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
3695
0
switch (RetVT.SimpleTy) {
3696
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
3697
0
  case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
3698
0
  default: return 0;
3699
0
}
3700
0
}
3701
3702
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
3703
0
  if (RetVT.SimpleTy != MVT::v32i16)
3704
0
    return 0;
3705
0
  if ((Subtarget->hasFP16())) {
3706
0
    return fastEmitInst_r(X86::VCVTPH2WZrr, &X86::VR512RegClass, Op0);
3707
0
  }
3708
0
  return 0;
3709
0
}
3710
3711
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
3712
0
  if ((Subtarget->hasVLX())) {
3713
0
    return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0);
3714
0
  }
3715
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3716
0
    return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0);
3717
0
  }
3718
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3719
0
    return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0);
3720
0
  }
3721
0
  return 0;
3722
0
}
3723
3724
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
3725
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3726
0
    return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0);
3727
0
  }
3728
0
  return 0;
3729
0
}
3730
3731
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
3732
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3733
0
    return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0);
3734
0
  }
3735
0
  return 0;
3736
0
}
3737
3738
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3739
0
switch (RetVT.SimpleTy) {
3740
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
3741
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
3742
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
3743
0
  default: return 0;
3744
0
}
3745
0
}
3746
3747
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
3748
0
  if ((Subtarget->hasVLX())) {
3749
0
    return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0);
3750
0
  }
3751
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3752
0
    return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0);
3753
0
  }
3754
0
  return 0;
3755
0
}
3756
3757
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
3758
0
  if ((Subtarget->hasDQI())) {
3759
0
    return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0);
3760
0
  }
3761
0
  return 0;
3762
0
}
3763
3764
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
3765
0
switch (RetVT.SimpleTy) {
3766
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
3767
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
3768
0
  default: return 0;
3769
0
}
3770
0
}
3771
3772
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
3773
0
  if (RetVT.SimpleTy != MVT::v16i32)
3774
0
    return 0;
3775
0
  if ((Subtarget->hasAVX512())) {
3776
0
    return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0);
3777
0
  }
3778
0
  return 0;
3779
0
}
3780
3781
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
3782
0
  if ((Subtarget->hasVLX())) {
3783
0
    return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0);
3784
0
  }
3785
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3786
0
    return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0);
3787
0
  }
3788
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3789
0
    return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0);
3790
0
  }
3791
0
  return 0;
3792
0
}
3793
3794
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
3795
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3796
0
    return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0);
3797
0
  }
3798
0
  return 0;
3799
0
}
3800
3801
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3802
0
switch (RetVT.SimpleTy) {
3803
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
3804
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
3805
0
  default: return 0;
3806
0
}
3807
0
}
3808
3809
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
3810
0
  if ((Subtarget->hasVLX())) {
3811
0
    return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0);
3812
0
  }
3813
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3814
0
    return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0);
3815
0
  }
3816
0
  return 0;
3817
0
}
3818
3819
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
3820
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3821
0
    return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0);
3822
0
  }
3823
0
  return 0;
3824
0
}
3825
3826
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
3827
0
switch (RetVT.SimpleTy) {
3828
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
3829
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
3830
0
  default: return 0;
3831
0
}
3832
0
}
3833
3834
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
3835
0
  if ((Subtarget->hasAVX512())) {
3836
0
    return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0);
3837
0
  }
3838
0
  return 0;
3839
0
}
3840
3841
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
3842
0
  if ((Subtarget->hasDQI())) {
3843
0
    return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0);
3844
0
  }
3845
0
  return 0;
3846
0
}
3847
3848
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
3849
0
switch (RetVT.SimpleTy) {
3850
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
3851
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
3852
0
  default: return 0;
3853
0
}
3854
0
}
3855
3856
0
unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
3857
0
  switch (VT.SimpleTy) {
3858
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0);
3859
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0);
3860
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0);
3861
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0);
3862
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0);
3863
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0);
3864
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0);
3865
0
  case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0);
3866
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0);
3867
0
  default: return 0;
3868
0
  }
3869
0
}
3870
3871
// FastEmit functions for X86ISD::CVTP2UI.
3872
3873
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
3874
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3875
0
    return fastEmitInst_r(X86::VCVTPH2UWZ128rr, &X86::VR128XRegClass, Op0);
3876
0
  }
3877
0
  return 0;
3878
0
}
3879
3880
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
3881
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3882
0
    return fastEmitInst_r(X86::VCVTPH2UDQZ128rr, &X86::VR128XRegClass, Op0);
3883
0
  }
3884
0
  return 0;
3885
0
}
3886
3887
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
3888
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3889
0
    return fastEmitInst_r(X86::VCVTPH2UDQZ256rr, &X86::VR256XRegClass, Op0);
3890
0
  }
3891
0
  return 0;
3892
0
}
3893
3894
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
3895
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3896
0
    return fastEmitInst_r(X86::VCVTPH2UQQZ128rr, &X86::VR128XRegClass, Op0);
3897
0
  }
3898
0
  return 0;
3899
0
}
3900
3901
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
3902
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3903
0
    return fastEmitInst_r(X86::VCVTPH2UQQZ256rr, &X86::VR256XRegClass, Op0);
3904
0
  }
3905
0
  return 0;
3906
0
}
3907
3908
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
3909
0
  if ((Subtarget->hasFP16())) {
3910
0
    return fastEmitInst_r(X86::VCVTPH2UQQZrr, &X86::VR512RegClass, Op0);
3911
0
  }
3912
0
  return 0;
3913
0
}
3914
3915
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3916
0
switch (RetVT.SimpleTy) {
3917
0
  case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
3918
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
3919
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
3920
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
3921
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
3922
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
3923
0
  default: return 0;
3924
0
}
3925
0
}
3926
3927
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
3928
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
3929
0
    return fastEmitInst_r(X86::VCVTPH2UWZ256rr, &X86::VR256XRegClass, Op0);
3930
0
  }
3931
0
  return 0;
3932
0
}
3933
3934
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
3935
0
  if ((Subtarget->hasFP16())) {
3936
0
    return fastEmitInst_r(X86::VCVTPH2UDQZrr, &X86::VR512RegClass, Op0);
3937
0
  }
3938
0
  return 0;
3939
0
}
3940
3941
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
3942
0
switch (RetVT.SimpleTy) {
3943
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
3944
0
  case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
3945
0
  default: return 0;
3946
0
}
3947
0
}
3948
3949
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
3950
0
  if (RetVT.SimpleTy != MVT::v32i16)
3951
0
    return 0;
3952
0
  if ((Subtarget->hasFP16())) {
3953
0
    return fastEmitInst_r(X86::VCVTPH2UWZrr, &X86::VR512RegClass, Op0);
3954
0
  }
3955
0
  return 0;
3956
0
}
3957
3958
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
3959
0
  if ((Subtarget->hasVLX())) {
3960
0
    return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0);
3961
0
  }
3962
0
  return 0;
3963
0
}
3964
3965
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
3966
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3967
0
    return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0);
3968
0
  }
3969
0
  return 0;
3970
0
}
3971
3972
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
3973
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3974
0
    return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0);
3975
0
  }
3976
0
  return 0;
3977
0
}
3978
3979
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3980
0
switch (RetVT.SimpleTy) {
3981
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
3982
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
3983
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
3984
0
  default: return 0;
3985
0
}
3986
0
}
3987
3988
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
3989
0
  if ((Subtarget->hasVLX())) {
3990
0
    return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0);
3991
0
  }
3992
0
  return 0;
3993
0
}
3994
3995
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
3996
0
  if ((Subtarget->hasDQI())) {
3997
0
    return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0);
3998
0
  }
3999
0
  return 0;
4000
0
}
4001
4002
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4003
0
switch (RetVT.SimpleTy) {
4004
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4005
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4006
0
  default: return 0;
4007
0
}
4008
0
}
4009
4010
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4011
0
  if (RetVT.SimpleTy != MVT::v16i32)
4012
0
    return 0;
4013
0
  if ((Subtarget->hasAVX512())) {
4014
0
    return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0);
4015
0
  }
4016
0
  return 0;
4017
0
}
4018
4019
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
4020
0
  if ((Subtarget->hasVLX())) {
4021
0
    return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0);
4022
0
  }
4023
0
  return 0;
4024
0
}
4025
4026
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
4027
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4028
0
    return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0);
4029
0
  }
4030
0
  return 0;
4031
0
}
4032
4033
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4034
0
switch (RetVT.SimpleTy) {
4035
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4036
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4037
0
  default: return 0;
4038
0
}
4039
0
}
4040
4041
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4042
0
  if ((Subtarget->hasVLX())) {
4043
0
    return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0);
4044
0
  }
4045
0
  return 0;
4046
0
}
4047
4048
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4049
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4050
0
    return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0);
4051
0
  }
4052
0
  return 0;
4053
0
}
4054
4055
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4056
0
switch (RetVT.SimpleTy) {
4057
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4058
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4059
0
  default: return 0;
4060
0
}
4061
0
}
4062
4063
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4064
0
  if ((Subtarget->hasAVX512())) {
4065
0
    return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0);
4066
0
  }
4067
0
  return 0;
4068
0
}
4069
4070
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4071
0
  if ((Subtarget->hasDQI())) {
4072
0
    return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0);
4073
0
  }
4074
0
  return 0;
4075
0
}
4076
4077
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4078
0
switch (RetVT.SimpleTy) {
4079
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4080
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4081
0
  default: return 0;
4082
0
}
4083
0
}
4084
4085
0
unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
4086
0
  switch (VT.SimpleTy) {
4087
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0);
4088
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0);
4089
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0);
4090
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0);
4091
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0);
4092
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0);
4093
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0);
4094
0
  case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0);
4095
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0);
4096
0
  default: return 0;
4097
0
  }
4098
0
}
4099
4100
// FastEmit functions for X86ISD::CVTPH2PS.
4101
4102
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
4103
0
  if ((Subtarget->hasVLX())) {
4104
0
    return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0);
4105
0
  }
4106
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4107
0
    return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0);
4108
0
  }
4109
0
  return 0;
4110
0
}
4111
4112
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) {
4113
0
  if ((Subtarget->hasVLX())) {
4114
0
    return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0);
4115
0
  }
4116
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
4117
0
    return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0);
4118
0
  }
4119
0
  return 0;
4120
0
}
4121
4122
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
4123
0
switch (RetVT.SimpleTy) {
4124
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
4125
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
4126
0
  default: return 0;
4127
0
}
4128
0
}
4129
4130
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
4131
0
  if (RetVT.SimpleTy != MVT::v16f32)
4132
0
    return 0;
4133
0
  if ((Subtarget->hasAVX512())) {
4134
0
    return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0);
4135
0
  }
4136
0
  return 0;
4137
0
}
4138
4139
0
unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) {
4140
0
  switch (VT.SimpleTy) {
4141
0
  case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
4142
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
4143
0
  default: return 0;
4144
0
  }
4145
0
}
4146
4147
// FastEmit functions for X86ISD::CVTPH2PS_SAE.
4148
4149
0
unsigned fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
4150
0
  if (RetVT.SimpleTy != MVT::v16f32)
4151
0
    return 0;
4152
0
  if ((Subtarget->hasAVX512())) {
4153
0
    return fastEmitInst_r(X86::VCVTPH2PSZrrb, &X86::VR512RegClass, Op0);
4154
0
  }
4155
0
  return 0;
4156
0
}
4157
4158
0
unsigned fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
4159
0
  switch (VT.SimpleTy) {
4160
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0);
4161
0
  default: return 0;
4162
0
  }
4163
0
}
4164
4165
// FastEmit functions for X86ISD::CVTS2SI.
4166
4167
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
4168
0
  if ((Subtarget->hasFP16())) {
4169
0
    return fastEmitInst_r(X86::VCVTSH2SIZrr_Int, &X86::GR32RegClass, Op0);
4170
0
  }
4171
0
  return 0;
4172
0
}
4173
4174
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
4175
0
  if ((Subtarget->hasFP16())) {
4176
0
    return fastEmitInst_r(X86::VCVTSH2SI64Zrr_Int, &X86::GR64RegClass, Op0);
4177
0
  }
4178
0
  return 0;
4179
0
}
4180
4181
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4182
0
switch (RetVT.SimpleTy) {
4183
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0);
4184
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0);
4185
0
  default: return 0;
4186
0
}
4187
0
}
4188
4189
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
4190
0
  if ((Subtarget->hasAVX512())) {
4191
0
    return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0);
4192
0
  }
4193
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4194
0
    return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0);
4195
0
  }
4196
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4197
0
    return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0);
4198
0
  }
4199
0
  return 0;
4200
0
}
4201
4202
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
4203
0
  if ((Subtarget->hasAVX512())) {
4204
0
    return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0);
4205
0
  }
4206
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4207
0
    return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0);
4208
0
  }
4209
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4210
0
    return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0);
4211
0
  }
4212
0
  return 0;
4213
0
}
4214
4215
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4216
0
switch (RetVT.SimpleTy) {
4217
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0);
4218
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0);
4219
0
  default: return 0;
4220
0
}
4221
0
}
4222
4223
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
4224
0
  if ((Subtarget->hasAVX512())) {
4225
0
    return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0);
4226
0
  }
4227
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4228
0
    return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0);
4229
0
  }
4230
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4231
0
    return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0);
4232
0
  }
4233
0
  return 0;
4234
0
}
4235
4236
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
4237
0
  if ((Subtarget->hasAVX512())) {
4238
0
    return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0);
4239
0
  }
4240
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4241
0
    return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0);
4242
0
  }
4243
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4244
0
    return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0);
4245
0
  }
4246
0
  return 0;
4247
0
}
4248
4249
0
unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4250
0
switch (RetVT.SimpleTy) {
4251
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0);
4252
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0);
4253
0
  default: return 0;
4254
0
}
4255
0
}
4256
4257
0
unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
4258
0
  switch (VT.SimpleTy) {
4259
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0);
4260
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0);
4261
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0);
4262
0
  default: return 0;
4263
0
  }
4264
0
}
4265
4266
// FastEmit functions for X86ISD::CVTS2UI.
4267
4268
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
4269
0
  if ((Subtarget->hasFP16())) {
4270
0
    return fastEmitInst_r(X86::VCVTSH2USIZrr_Int, &X86::GR32RegClass, Op0);
4271
0
  }
4272
0
  return 0;
4273
0
}
4274
4275
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
4276
0
  if ((Subtarget->hasFP16())) {
4277
0
    return fastEmitInst_r(X86::VCVTSH2USI64Zrr_Int, &X86::GR64RegClass, Op0);
4278
0
  }
4279
0
  return 0;
4280
0
}
4281
4282
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4283
0
switch (RetVT.SimpleTy) {
4284
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0);
4285
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0);
4286
0
  default: return 0;
4287
0
}
4288
0
}
4289
4290
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
4291
0
  if ((Subtarget->hasAVX512())) {
4292
0
    return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0);
4293
0
  }
4294
0
  return 0;
4295
0
}
4296
4297
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
4298
0
  if ((Subtarget->hasAVX512())) {
4299
0
    return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0);
4300
0
  }
4301
0
  return 0;
4302
0
}
4303
4304
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4305
0
switch (RetVT.SimpleTy) {
4306
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0);
4307
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0);
4308
0
  default: return 0;
4309
0
}
4310
0
}
4311
4312
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
4313
0
  if ((Subtarget->hasAVX512())) {
4314
0
    return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0);
4315
0
  }
4316
0
  return 0;
4317
0
}
4318
4319
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
4320
0
  if ((Subtarget->hasAVX512())) {
4321
0
    return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0);
4322
0
  }
4323
0
  return 0;
4324
0
}
4325
4326
0
unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4327
0
switch (RetVT.SimpleTy) {
4328
0
  case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0);
4329
0
  case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0);
4330
0
  default: return 0;
4331
0
}
4332
0
}
4333
4334
0
unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
4335
0
  switch (VT.SimpleTy) {
4336
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0);
4337
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0);
4338
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0);
4339
0
  default: return 0;
4340
0
  }
4341
0
}
4342
4343
// FastEmit functions for X86ISD::CVTSI2P.
4344
4345
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
4346
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4347
0
    return fastEmitInst_r(X86::VCVTDQ2PHZ128rr, &X86::VR128XRegClass, Op0);
4348
0
  }
4349
0
  return 0;
4350
0
}
4351
4352
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
4353
0
  if ((Subtarget->hasVLX())) {
4354
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0);
4355
0
  }
4356
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4357
0
    return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0);
4358
0
  }
4359
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4360
0
    return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0);
4361
0
  }
4362
0
  return 0;
4363
0
}
4364
4365
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
4366
0
switch (RetVT.SimpleTy) {
4367
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
4368
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
4369
0
  default: return 0;
4370
0
}
4371
0
}
4372
4373
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
4374
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4375
0
    return fastEmitInst_r(X86::VCVTQQ2PHZ128rr, &X86::VR128XRegClass, Op0);
4376
0
  }
4377
0
  return 0;
4378
0
}
4379
4380
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
4381
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4382
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0);
4383
0
  }
4384
0
  return 0;
4385
0
}
4386
4387
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
4388
0
switch (RetVT.SimpleTy) {
4389
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
4390
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
4391
0
  default: return 0;
4392
0
}
4393
0
}
4394
4395
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
4396
0
  if (RetVT.SimpleTy != MVT::v8f16)
4397
0
    return 0;
4398
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4399
0
    return fastEmitInst_r(X86::VCVTQQ2PHZ256rr, &X86::VR128XRegClass, Op0);
4400
0
  }
4401
0
  return 0;
4402
0
}
4403
4404
0
unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
4405
0
  switch (VT.SimpleTy) {
4406
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
4407
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
4408
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
4409
0
  default: return 0;
4410
0
  }
4411
0
}
4412
4413
// FastEmit functions for X86ISD::CVTTP2SI.
4414
4415
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
4416
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4417
0
    return fastEmitInst_r(X86::VCVTTPH2WZ128rr, &X86::VR128XRegClass, Op0);
4418
0
  }
4419
0
  return 0;
4420
0
}
4421
4422
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
4423
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4424
0
    return fastEmitInst_r(X86::VCVTTPH2DQZ128rr, &X86::VR128XRegClass, Op0);
4425
0
  }
4426
0
  return 0;
4427
0
}
4428
4429
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
4430
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4431
0
    return fastEmitInst_r(X86::VCVTTPH2DQZ256rr, &X86::VR256XRegClass, Op0);
4432
0
  }
4433
0
  return 0;
4434
0
}
4435
4436
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
4437
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4438
0
    return fastEmitInst_r(X86::VCVTTPH2QQZ128rr, &X86::VR128XRegClass, Op0);
4439
0
  }
4440
0
  return 0;
4441
0
}
4442
4443
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
4444
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4445
0
    return fastEmitInst_r(X86::VCVTTPH2QQZ256rr, &X86::VR256XRegClass, Op0);
4446
0
  }
4447
0
  return 0;
4448
0
}
4449
4450
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
4451
0
  if ((Subtarget->hasFP16())) {
4452
0
    return fastEmitInst_r(X86::VCVTTPH2QQZrr, &X86::VR512RegClass, Op0);
4453
0
  }
4454
0
  return 0;
4455
0
}
4456
4457
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4458
0
switch (RetVT.SimpleTy) {
4459
0
  case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
4460
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
4461
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
4462
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
4463
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
4464
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
4465
0
  default: return 0;
4466
0
}
4467
0
}
4468
4469
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
4470
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4471
0
    return fastEmitInst_r(X86::VCVTTPH2WZ256rr, &X86::VR256XRegClass, Op0);
4472
0
  }
4473
0
  return 0;
4474
0
}
4475
4476
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
4477
0
  if ((Subtarget->hasFP16())) {
4478
0
    return fastEmitInst_r(X86::VCVTTPH2DQZrr, &X86::VR512RegClass, Op0);
4479
0
  }
4480
0
  return 0;
4481
0
}
4482
4483
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4484
0
switch (RetVT.SimpleTy) {
4485
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
4486
0
  case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
4487
0
  default: return 0;
4488
0
}
4489
0
}
4490
4491
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4492
0
  if (RetVT.SimpleTy != MVT::v32i16)
4493
0
    return 0;
4494
0
  if ((Subtarget->hasFP16())) {
4495
0
    return fastEmitInst_r(X86::VCVTTPH2WZrr, &X86::VR512RegClass, Op0);
4496
0
  }
4497
0
  return 0;
4498
0
}
4499
4500
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
4501
0
  if ((Subtarget->hasVLX())) {
4502
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0);
4503
0
  }
4504
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4505
0
    return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0);
4506
0
  }
4507
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4508
0
    return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0);
4509
0
  }
4510
0
  return 0;
4511
0
}
4512
4513
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
4514
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4515
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0);
4516
0
  }
4517
0
  return 0;
4518
0
}
4519
4520
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
4521
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4522
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0);
4523
0
  }
4524
0
  return 0;
4525
0
}
4526
4527
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4528
0
switch (RetVT.SimpleTy) {
4529
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
4530
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
4531
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
4532
0
  default: return 0;
4533
0
}
4534
0
}
4535
4536
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
4537
0
  if ((Subtarget->hasVLX())) {
4538
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0);
4539
0
  }
4540
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4541
0
    return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0);
4542
0
  }
4543
0
  return 0;
4544
0
}
4545
4546
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
4547
0
  if ((Subtarget->hasDQI())) {
4548
0
    return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0);
4549
0
  }
4550
0
  return 0;
4551
0
}
4552
4553
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4554
0
switch (RetVT.SimpleTy) {
4555
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
4556
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
4557
0
  default: return 0;
4558
0
}
4559
0
}
4560
4561
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4562
0
  if (RetVT.SimpleTy != MVT::v16i32)
4563
0
    return 0;
4564
0
  if ((Subtarget->hasAVX512())) {
4565
0
    return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0);
4566
0
  }
4567
0
  return 0;
4568
0
}
4569
4570
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
4571
0
  if ((Subtarget->hasVLX())) {
4572
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0);
4573
0
  }
4574
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4575
0
    return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0);
4576
0
  }
4577
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4578
0
    return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0);
4579
0
  }
4580
0
  return 0;
4581
0
}
4582
4583
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
4584
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4585
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0);
4586
0
  }
4587
0
  return 0;
4588
0
}
4589
4590
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4591
0
switch (RetVT.SimpleTy) {
4592
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
4593
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
4594
0
  default: return 0;
4595
0
}
4596
0
}
4597
4598
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4599
0
  if ((Subtarget->hasVLX())) {
4600
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0);
4601
0
  }
4602
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4603
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0);
4604
0
  }
4605
0
  return 0;
4606
0
}
4607
4608
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4609
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4610
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0);
4611
0
  }
4612
0
  return 0;
4613
0
}
4614
4615
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4616
0
switch (RetVT.SimpleTy) {
4617
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
4618
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
4619
0
  default: return 0;
4620
0
}
4621
0
}
4622
4623
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4624
0
  if ((Subtarget->hasAVX512())) {
4625
0
    return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0);
4626
0
  }
4627
0
  return 0;
4628
0
}
4629
4630
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4631
0
  if ((Subtarget->hasDQI())) {
4632
0
    return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0);
4633
0
  }
4634
0
  return 0;
4635
0
}
4636
4637
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4638
0
switch (RetVT.SimpleTy) {
4639
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
4640
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
4641
0
  default: return 0;
4642
0
}
4643
0
}
4644
4645
0
unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
4646
0
  switch (VT.SimpleTy) {
4647
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
4648
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
4649
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
4650
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
4651
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
4652
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
4653
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
4654
0
  case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
4655
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
4656
0
  default: return 0;
4657
0
  }
4658
0
}
4659
4660
// FastEmit functions for X86ISD::CVTTP2SI_SAE.
4661
4662
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4663
0
  if (RetVT.SimpleTy != MVT::v8i64)
4664
0
    return 0;
4665
0
  if ((Subtarget->hasFP16())) {
4666
0
    return fastEmitInst_r(X86::VCVTTPH2QQZrrb, &X86::VR512RegClass, Op0);
4667
0
  }
4668
0
  return 0;
4669
0
}
4670
4671
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4672
0
  if (RetVT.SimpleTy != MVT::v16i32)
4673
0
    return 0;
4674
0
  if ((Subtarget->hasFP16())) {
4675
0
    return fastEmitInst_r(X86::VCVTTPH2DQZrrb, &X86::VR512RegClass, Op0);
4676
0
  }
4677
0
  return 0;
4678
0
}
4679
4680
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4681
0
  if (RetVT.SimpleTy != MVT::v32i16)
4682
0
    return 0;
4683
0
  if ((Subtarget->hasFP16())) {
4684
0
    return fastEmitInst_r(X86::VCVTTPH2WZrrb, &X86::VR512RegClass, Op0);
4685
0
  }
4686
0
  return 0;
4687
0
}
4688
4689
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4690
0
  if (RetVT.SimpleTy != MVT::v8i64)
4691
0
    return 0;
4692
0
  if ((Subtarget->hasDQI())) {
4693
0
    return fastEmitInst_r(X86::VCVTTPS2QQZrrb, &X86::VR512RegClass, Op0);
4694
0
  }
4695
0
  return 0;
4696
0
}
4697
4698
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4699
0
  if (RetVT.SimpleTy != MVT::v16i32)
4700
0
    return 0;
4701
0
  if ((Subtarget->hasAVX512())) {
4702
0
    return fastEmitInst_r(X86::VCVTTPS2DQZrrb, &X86::VR512RegClass, Op0);
4703
0
  }
4704
0
  return 0;
4705
0
}
4706
4707
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4708
0
  if ((Subtarget->hasAVX512())) {
4709
0
    return fastEmitInst_r(X86::VCVTTPD2DQZrrb, &X86::VR256XRegClass, Op0);
4710
0
  }
4711
0
  return 0;
4712
0
}
4713
4714
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4715
0
  if ((Subtarget->hasDQI())) {
4716
0
    return fastEmitInst_r(X86::VCVTTPD2QQZrrb, &X86::VR512RegClass, Op0);
4717
0
  }
4718
0
  return 0;
4719
0
}
4720
4721
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4722
0
switch (RetVT.SimpleTy) {
4723
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
4724
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
4725
0
  default: return 0;
4726
0
}
4727
0
}
4728
4729
0
unsigned fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
4730
0
  switch (VT.SimpleTy) {
4731
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0);
4732
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0);
4733
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0);
4734
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0);
4735
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0);
4736
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0);
4737
0
  default: return 0;
4738
0
  }
4739
0
}
4740
4741
// FastEmit functions for X86ISD::CVTTP2UI.
4742
4743
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
4744
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4745
0
    return fastEmitInst_r(X86::VCVTTPH2UWZ128rr, &X86::VR128XRegClass, Op0);
4746
0
  }
4747
0
  return 0;
4748
0
}
4749
4750
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
4751
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4752
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZ128rr, &X86::VR128XRegClass, Op0);
4753
0
  }
4754
0
  return 0;
4755
0
}
4756
4757
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
4758
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4759
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZ256rr, &X86::VR256XRegClass, Op0);
4760
0
  }
4761
0
  return 0;
4762
0
}
4763
4764
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
4765
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4766
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZ128rr, &X86::VR128XRegClass, Op0);
4767
0
  }
4768
0
  return 0;
4769
0
}
4770
4771
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
4772
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4773
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZ256rr, &X86::VR256XRegClass, Op0);
4774
0
  }
4775
0
  return 0;
4776
0
}
4777
4778
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
4779
0
  if ((Subtarget->hasFP16())) {
4780
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZrr, &X86::VR512RegClass, Op0);
4781
0
  }
4782
0
  return 0;
4783
0
}
4784
4785
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4786
0
switch (RetVT.SimpleTy) {
4787
0
  case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
4788
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
4789
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
4790
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
4791
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
4792
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
4793
0
  default: return 0;
4794
0
}
4795
0
}
4796
4797
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
4798
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
4799
0
    return fastEmitInst_r(X86::VCVTTPH2UWZ256rr, &X86::VR256XRegClass, Op0);
4800
0
  }
4801
0
  return 0;
4802
0
}
4803
4804
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
4805
0
  if ((Subtarget->hasFP16())) {
4806
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZrr, &X86::VR512RegClass, Op0);
4807
0
  }
4808
0
  return 0;
4809
0
}
4810
4811
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4812
0
switch (RetVT.SimpleTy) {
4813
0
  case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
4814
0
  case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
4815
0
  default: return 0;
4816
0
}
4817
0
}
4818
4819
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4820
0
  if (RetVT.SimpleTy != MVT::v32i16)
4821
0
    return 0;
4822
0
  if ((Subtarget->hasFP16())) {
4823
0
    return fastEmitInst_r(X86::VCVTTPH2UWZrr, &X86::VR512RegClass, Op0);
4824
0
  }
4825
0
  return 0;
4826
0
}
4827
4828
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
4829
0
  if ((Subtarget->hasVLX())) {
4830
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0);
4831
0
  }
4832
0
  return 0;
4833
0
}
4834
4835
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
4836
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4837
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0);
4838
0
  }
4839
0
  return 0;
4840
0
}
4841
4842
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
4843
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4844
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0);
4845
0
  }
4846
0
  return 0;
4847
0
}
4848
4849
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4850
0
switch (RetVT.SimpleTy) {
4851
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
4852
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
4853
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
4854
0
  default: return 0;
4855
0
}
4856
0
}
4857
4858
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
4859
0
  if ((Subtarget->hasVLX())) {
4860
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0);
4861
0
  }
4862
0
  return 0;
4863
0
}
4864
4865
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
4866
0
  if ((Subtarget->hasDQI())) {
4867
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0);
4868
0
  }
4869
0
  return 0;
4870
0
}
4871
4872
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
4873
0
switch (RetVT.SimpleTy) {
4874
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
4875
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
4876
0
  default: return 0;
4877
0
}
4878
0
}
4879
4880
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
4881
0
  if (RetVT.SimpleTy != MVT::v16i32)
4882
0
    return 0;
4883
0
  if ((Subtarget->hasAVX512())) {
4884
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0);
4885
0
  }
4886
0
  return 0;
4887
0
}
4888
4889
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
4890
0
  if ((Subtarget->hasVLX())) {
4891
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0);
4892
0
  }
4893
0
  return 0;
4894
0
}
4895
4896
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
4897
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4898
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0);
4899
0
  }
4900
0
  return 0;
4901
0
}
4902
4903
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4904
0
switch (RetVT.SimpleTy) {
4905
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
4906
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
4907
0
  default: return 0;
4908
0
}
4909
0
}
4910
4911
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
4912
0
  if ((Subtarget->hasVLX())) {
4913
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0);
4914
0
  }
4915
0
  return 0;
4916
0
}
4917
4918
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
4919
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4920
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0);
4921
0
  }
4922
0
  return 0;
4923
0
}
4924
4925
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
4926
0
switch (RetVT.SimpleTy) {
4927
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
4928
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
4929
0
  default: return 0;
4930
0
}
4931
0
}
4932
4933
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
4934
0
  if ((Subtarget->hasAVX512())) {
4935
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0);
4936
0
  }
4937
0
  return 0;
4938
0
}
4939
4940
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
4941
0
  if ((Subtarget->hasDQI())) {
4942
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0);
4943
0
  }
4944
0
  return 0;
4945
0
}
4946
4947
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
4948
0
switch (RetVT.SimpleTy) {
4949
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
4950
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
4951
0
  default: return 0;
4952
0
}
4953
0
}
4954
4955
0
unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
4956
0
  switch (VT.SimpleTy) {
4957
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
4958
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
4959
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
4960
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
4961
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
4962
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
4963
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
4964
0
  case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
4965
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
4966
0
  default: return 0;
4967
0
  }
4968
0
}
4969
4970
// FastEmit functions for X86ISD::CVTTP2UI_SAE.
4971
4972
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4973
0
  if (RetVT.SimpleTy != MVT::v8i64)
4974
0
    return 0;
4975
0
  if ((Subtarget->hasFP16())) {
4976
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZrrb, &X86::VR512RegClass, Op0);
4977
0
  }
4978
0
  return 0;
4979
0
}
4980
4981
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
4982
0
  if (RetVT.SimpleTy != MVT::v16i32)
4983
0
    return 0;
4984
0
  if ((Subtarget->hasFP16())) {
4985
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZrrb, &X86::VR512RegClass, Op0);
4986
0
  }
4987
0
  return 0;
4988
0
}
4989
4990
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
4991
0
  if (RetVT.SimpleTy != MVT::v32i16)
4992
0
    return 0;
4993
0
  if ((Subtarget->hasFP16())) {
4994
0
    return fastEmitInst_r(X86::VCVTTPH2UWZrrb, &X86::VR512RegClass, Op0);
4995
0
  }
4996
0
  return 0;
4997
0
}
4998
4999
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5000
0
  if (RetVT.SimpleTy != MVT::v8i64)
5001
0
    return 0;
5002
0
  if ((Subtarget->hasDQI())) {
5003
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZrrb, &X86::VR512RegClass, Op0);
5004
0
  }
5005
0
  return 0;
5006
0
}
5007
5008
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5009
0
  if (RetVT.SimpleTy != MVT::v16i32)
5010
0
    return 0;
5011
0
  if ((Subtarget->hasAVX512())) {
5012
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZrrb, &X86::VR512RegClass, Op0);
5013
0
  }
5014
0
  return 0;
5015
0
}
5016
5017
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
5018
0
  if ((Subtarget->hasAVX512())) {
5019
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZrrb, &X86::VR256XRegClass, Op0);
5020
0
  }
5021
0
  return 0;
5022
0
}
5023
5024
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
5025
0
  if ((Subtarget->hasDQI())) {
5026
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZrrb, &X86::VR512RegClass, Op0);
5027
0
  }
5028
0
  return 0;
5029
0
}
5030
5031
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5032
0
switch (RetVT.SimpleTy) {
5033
0
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0);
5034
0
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0);
5035
0
  default: return 0;
5036
0
}
5037
0
}
5038
5039
0
unsigned fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5040
0
  switch (VT.SimpleTy) {
5041
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0);
5042
0
  case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0);
5043
0
  case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0);
5044
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0);
5045
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0);
5046
0
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0);
5047
0
  default: return 0;
5048
0
  }
5049
0
}
5050
5051
// FastEmit functions for X86ISD::CVTTS2SI.
5052
5053
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5054
0
  if ((Subtarget->hasFP16())) {
5055
0
    return fastEmitInst_r(X86::VCVTTSH2SIZrr_Int, &X86::GR32RegClass, Op0);
5056
0
  }
5057
0
  return 0;
5058
0
}
5059
5060
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5061
0
  if ((Subtarget->hasFP16())) {
5062
0
    return fastEmitInst_r(X86::VCVTTSH2SI64Zrr_Int, &X86::GR64RegClass, Op0);
5063
0
  }
5064
0
  return 0;
5065
0
}
5066
5067
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5068
0
switch (RetVT.SimpleTy) {
5069
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0);
5070
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0);
5071
0
  default: return 0;
5072
0
}
5073
0
}
5074
5075
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5076
0
  if ((Subtarget->hasAVX512())) {
5077
0
    return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0);
5078
0
  }
5079
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5080
0
    return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0);
5081
0
  }
5082
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5083
0
    return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0);
5084
0
  }
5085
0
  return 0;
5086
0
}
5087
5088
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5089
0
  if ((Subtarget->hasAVX512())) {
5090
0
    return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0);
5091
0
  }
5092
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5093
0
    return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0);
5094
0
  }
5095
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5096
0
    return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0);
5097
0
  }
5098
0
  return 0;
5099
0
}
5100
5101
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5102
0
switch (RetVT.SimpleTy) {
5103
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0);
5104
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0);
5105
0
  default: return 0;
5106
0
}
5107
0
}
5108
5109
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5110
0
  if ((Subtarget->hasAVX512())) {
5111
0
    return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0);
5112
0
  }
5113
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5114
0
    return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0);
5115
0
  }
5116
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5117
0
    return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0);
5118
0
  }
5119
0
  return 0;
5120
0
}
5121
5122
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5123
0
  if ((Subtarget->hasAVX512())) {
5124
0
    return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0);
5125
0
  }
5126
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5127
0
    return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0);
5128
0
  }
5129
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5130
0
    return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0);
5131
0
  }
5132
0
  return 0;
5133
0
}
5134
5135
0
unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5136
0
switch (RetVT.SimpleTy) {
5137
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0);
5138
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0);
5139
0
  default: return 0;
5140
0
}
5141
0
}
5142
5143
0
unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
5144
0
  switch (VT.SimpleTy) {
5145
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0);
5146
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0);
5147
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0);
5148
0
  default: return 0;
5149
0
  }
5150
0
}
5151
5152
// FastEmit functions for X86ISD::CVTTS2SI_SAE.
5153
5154
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5155
0
  if ((Subtarget->hasFP16())) {
5156
0
    return fastEmitInst_r(X86::VCVTTSH2SIZrrb_Int, &X86::GR32RegClass, Op0);
5157
0
  }
5158
0
  return 0;
5159
0
}
5160
5161
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5162
0
  if ((Subtarget->hasFP16())) {
5163
0
    return fastEmitInst_r(X86::VCVTTSH2SI64Zrrb_Int, &X86::GR64RegClass, Op0);
5164
0
  }
5165
0
  return 0;
5166
0
}
5167
5168
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5169
0
switch (RetVT.SimpleTy) {
5170
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0);
5171
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0);
5172
0
  default: return 0;
5173
0
}
5174
0
}
5175
5176
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5177
0
  if ((Subtarget->hasAVX512())) {
5178
0
    return fastEmitInst_r(X86::VCVTTSS2SIZrrb_Int, &X86::GR32RegClass, Op0);
5179
0
  }
5180
0
  return 0;
5181
0
}
5182
5183
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5184
0
  if ((Subtarget->hasAVX512())) {
5185
0
    return fastEmitInst_r(X86::VCVTTSS2SI64Zrrb_Int, &X86::GR64RegClass, Op0);
5186
0
  }
5187
0
  return 0;
5188
0
}
5189
5190
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5191
0
switch (RetVT.SimpleTy) {
5192
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0);
5193
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0);
5194
0
  default: return 0;
5195
0
}
5196
0
}
5197
5198
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5199
0
  if ((Subtarget->hasAVX512())) {
5200
0
    return fastEmitInst_r(X86::VCVTTSD2SIZrrb_Int, &X86::GR32RegClass, Op0);
5201
0
  }
5202
0
  return 0;
5203
0
}
5204
5205
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5206
0
  if ((Subtarget->hasAVX512())) {
5207
0
    return fastEmitInst_r(X86::VCVTTSD2SI64Zrrb_Int, &X86::GR64RegClass, Op0);
5208
0
  }
5209
0
  return 0;
5210
0
}
5211
5212
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5213
0
switch (RetVT.SimpleTy) {
5214
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0);
5215
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0);
5216
0
  default: return 0;
5217
0
}
5218
0
}
5219
5220
0
unsigned fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5221
0
  switch (VT.SimpleTy) {
5222
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0);
5223
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0);
5224
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0);
5225
0
  default: return 0;
5226
0
  }
5227
0
}
5228
5229
// FastEmit functions for X86ISD::CVTTS2UI.
5230
5231
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5232
0
  if ((Subtarget->hasFP16())) {
5233
0
    return fastEmitInst_r(X86::VCVTTSH2USIZrr_Int, &X86::GR32RegClass, Op0);
5234
0
  }
5235
0
  return 0;
5236
0
}
5237
5238
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5239
0
  if ((Subtarget->hasFP16())) {
5240
0
    return fastEmitInst_r(X86::VCVTTSH2USI64Zrr_Int, &X86::GR64RegClass, Op0);
5241
0
  }
5242
0
  return 0;
5243
0
}
5244
5245
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5246
0
switch (RetVT.SimpleTy) {
5247
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0);
5248
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0);
5249
0
  default: return 0;
5250
0
}
5251
0
}
5252
5253
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5254
0
  if ((Subtarget->hasAVX512())) {
5255
0
    return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0);
5256
0
  }
5257
0
  return 0;
5258
0
}
5259
5260
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5261
0
  if ((Subtarget->hasAVX512())) {
5262
0
    return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0);
5263
0
  }
5264
0
  return 0;
5265
0
}
5266
5267
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5268
0
switch (RetVT.SimpleTy) {
5269
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0);
5270
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0);
5271
0
  default: return 0;
5272
0
}
5273
0
}
5274
5275
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5276
0
  if ((Subtarget->hasAVX512())) {
5277
0
    return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0);
5278
0
  }
5279
0
  return 0;
5280
0
}
5281
5282
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5283
0
  if ((Subtarget->hasAVX512())) {
5284
0
    return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0);
5285
0
  }
5286
0
  return 0;
5287
0
}
5288
5289
0
unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5290
0
switch (RetVT.SimpleTy) {
5291
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0);
5292
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0);
5293
0
  default: return 0;
5294
0
}
5295
0
}
5296
5297
0
unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
5298
0
  switch (VT.SimpleTy) {
5299
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0);
5300
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0);
5301
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0);
5302
0
  default: return 0;
5303
0
  }
5304
0
}
5305
5306
// FastEmit functions for X86ISD::CVTTS2UI_SAE.
5307
5308
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) {
5309
0
  if ((Subtarget->hasFP16())) {
5310
0
    return fastEmitInst_r(X86::VCVTTSH2USIZrrb_Int, &X86::GR32RegClass, Op0);
5311
0
  }
5312
0
  return 0;
5313
0
}
5314
5315
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) {
5316
0
  if ((Subtarget->hasFP16())) {
5317
0
    return fastEmitInst_r(X86::VCVTTSH2USI64Zrrb_Int, &X86::GR64RegClass, Op0);
5318
0
  }
5319
0
  return 0;
5320
0
}
5321
5322
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5323
0
switch (RetVT.SimpleTy) {
5324
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0);
5325
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0);
5326
0
  default: return 0;
5327
0
}
5328
0
}
5329
5330
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) {
5331
0
  if ((Subtarget->hasAVX512())) {
5332
0
    return fastEmitInst_r(X86::VCVTTSS2USIZrrb_Int, &X86::GR32RegClass, Op0);
5333
0
  }
5334
0
  return 0;
5335
0
}
5336
5337
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) {
5338
0
  if ((Subtarget->hasAVX512())) {
5339
0
    return fastEmitInst_r(X86::VCVTTSS2USI64Zrrb_Int, &X86::GR64RegClass, Op0);
5340
0
  }
5341
0
  return 0;
5342
0
}
5343
5344
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5345
0
switch (RetVT.SimpleTy) {
5346
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0);
5347
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0);
5348
0
  default: return 0;
5349
0
}
5350
0
}
5351
5352
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) {
5353
0
  if ((Subtarget->hasAVX512())) {
5354
0
    return fastEmitInst_r(X86::VCVTTSD2USIZrrb_Int, &X86::GR32RegClass, Op0);
5355
0
  }
5356
0
  return 0;
5357
0
}
5358
5359
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) {
5360
0
  if ((Subtarget->hasAVX512())) {
5361
0
    return fastEmitInst_r(X86::VCVTTSD2USI64Zrrb_Int, &X86::GR64RegClass, Op0);
5362
0
  }
5363
0
  return 0;
5364
0
}
5365
5366
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5367
0
switch (RetVT.SimpleTy) {
5368
0
  case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0);
5369
0
  case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0);
5370
0
  default: return 0;
5371
0
}
5372
0
}
5373
5374
0
unsigned fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5375
0
  switch (VT.SimpleTy) {
5376
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0);
5377
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0);
5378
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0);
5379
0
  default: return 0;
5380
0
  }
5381
0
}
5382
5383
// FastEmit functions for X86ISD::CVTUI2P.
5384
5385
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
5386
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5387
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZ128rr, &X86::VR128XRegClass, Op0);
5388
0
  }
5389
0
  return 0;
5390
0
}
5391
5392
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
5393
0
  if ((Subtarget->hasVLX())) {
5394
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0);
5395
0
  }
5396
0
  return 0;
5397
0
}
5398
5399
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
5400
0
switch (RetVT.SimpleTy) {
5401
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
5402
0
  case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
5403
0
  default: return 0;
5404
0
}
5405
0
}
5406
5407
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
5408
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5409
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZ128rr, &X86::VR128XRegClass, Op0);
5410
0
  }
5411
0
  return 0;
5412
0
}
5413
5414
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
5415
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5416
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0);
5417
0
  }
5418
0
  return 0;
5419
0
}
5420
5421
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
5422
0
switch (RetVT.SimpleTy) {
5423
0
  case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
5424
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
5425
0
  default: return 0;
5426
0
}
5427
0
}
5428
5429
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
5430
0
  if (RetVT.SimpleTy != MVT::v8f16)
5431
0
    return 0;
5432
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5433
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZ256rr, &X86::VR128XRegClass, Op0);
5434
0
  }
5435
0
  return 0;
5436
0
}
5437
5438
0
unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
5439
0
  switch (VT.SimpleTy) {
5440
0
  case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
5441
0
  case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
5442
0
  case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
5443
0
  default: return 0;
5444
0
  }
5445
0
}
5446
5447
// FastEmit functions for X86ISD::DYN_ALLOCA.
5448
5449
0
unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
5450
0
  if (RetVT.SimpleTy != MVT::isVoid)
5451
0
    return 0;
5452
0
  if ((!Subtarget->isTarget64BitLP64())) {
5453
0
    return fastEmitInst_r(X86::DYN_ALLOCA_32, &X86::GR32RegClass, Op0);
5454
0
  }
5455
0
  return 0;
5456
0
}
5457
5458
0
unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
5459
0
  if (RetVT.SimpleTy != MVT::isVoid)
5460
0
    return 0;
5461
0
  if ((Subtarget->is64Bit())) {
5462
0
    return fastEmitInst_r(X86::DYN_ALLOCA_64, &X86::GR64RegClass, Op0);
5463
0
  }
5464
0
  return 0;
5465
0
}
5466
5467
0
unsigned fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
5468
0
  switch (VT.SimpleTy) {
5469
0
  case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0);
5470
0
  case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0);
5471
0
  default: return 0;
5472
0
  }
5473
0
}
5474
5475
// FastEmit functions for X86ISD::EH_RETURN.
5476
5477
0
unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0) {
5478
0
  if (RetVT.SimpleTy != MVT::isVoid)
5479
0
    return 0;
5480
0
  return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0);
5481
0
}
5482
5483
0
unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0) {
5484
0
  if (RetVT.SimpleTy != MVT::isVoid)
5485
0
    return 0;
5486
0
  return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0);
5487
0
}
5488
5489
0
unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0) {
5490
0
  switch (VT.SimpleTy) {
5491
0
  case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0);
5492
0
  case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0);
5493
0
  default: return 0;
5494
0
  }
5495
0
}
5496
5497
// FastEmit functions for X86ISD::EXP2.
5498
5499
0
unsigned fastEmit_X86ISD_EXP2_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5500
0
  if (RetVT.SimpleTy != MVT::v16f32)
5501
0
    return 0;
5502
0
  if ((Subtarget->hasERI())) {
5503
0
    return fastEmitInst_r(X86::VEXP2PSZr, &X86::VR512RegClass, Op0);
5504
0
  }
5505
0
  return 0;
5506
0
}
5507
5508
0
unsigned fastEmit_X86ISD_EXP2_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5509
0
  if (RetVT.SimpleTy != MVT::v8f64)
5510
0
    return 0;
5511
0
  if ((Subtarget->hasERI())) {
5512
0
    return fastEmitInst_r(X86::VEXP2PDZr, &X86::VR512RegClass, Op0);
5513
0
  }
5514
0
  return 0;
5515
0
}
5516
5517
0
unsigned fastEmit_X86ISD_EXP2_r(MVT VT, MVT RetVT, unsigned Op0) {
5518
0
  switch (VT.SimpleTy) {
5519
0
  case MVT::v16f32: return fastEmit_X86ISD_EXP2_MVT_v16f32_r(RetVT, Op0);
5520
0
  case MVT::v8f64: return fastEmit_X86ISD_EXP2_MVT_v8f64_r(RetVT, Op0);
5521
0
  default: return 0;
5522
0
  }
5523
0
}
5524
5525
// FastEmit functions for X86ISD::EXP2_SAE.
5526
5527
0
unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5528
0
  if (RetVT.SimpleTy != MVT::v16f32)
5529
0
    return 0;
5530
0
  if ((Subtarget->hasERI())) {
5531
0
    return fastEmitInst_r(X86::VEXP2PSZrb, &X86::VR512RegClass, Op0);
5532
0
  }
5533
0
  return 0;
5534
0
}
5535
5536
0
unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5537
0
  if (RetVT.SimpleTy != MVT::v8f64)
5538
0
    return 0;
5539
0
  if ((Subtarget->hasERI())) {
5540
0
    return fastEmitInst_r(X86::VEXP2PDZrb, &X86::VR512RegClass, Op0);
5541
0
  }
5542
0
  return 0;
5543
0
}
5544
5545
0
unsigned fastEmit_X86ISD_EXP2_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5546
0
  switch (VT.SimpleTy) {
5547
0
  case MVT::v16f32: return fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(RetVT, Op0);
5548
0
  case MVT::v8f64: return fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(RetVT, Op0);
5549
0
  default: return 0;
5550
0
  }
5551
0
}
5552
5553
// FastEmit functions for X86ISD::FGETEXP.
5554
5555
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5556
0
  if (RetVT.SimpleTy != MVT::v8f16)
5557
0
    return 0;
5558
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5559
0
    return fastEmitInst_r(X86::VGETEXPPHZ128r, &X86::VR128XRegClass, Op0);
5560
0
  }
5561
0
  return 0;
5562
0
}
5563
5564
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
5565
0
  if (RetVT.SimpleTy != MVT::v16f16)
5566
0
    return 0;
5567
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
5568
0
    return fastEmitInst_r(X86::VGETEXPPHZ256r, &X86::VR256XRegClass, Op0);
5569
0
  }
5570
0
  return 0;
5571
0
}
5572
5573
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5574
0
  if (RetVT.SimpleTy != MVT::v32f16)
5575
0
    return 0;
5576
0
  if ((Subtarget->hasFP16())) {
5577
0
    return fastEmitInst_r(X86::VGETEXPPHZr, &X86::VR512RegClass, Op0);
5578
0
  }
5579
0
  return 0;
5580
0
}
5581
5582
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5583
0
  if (RetVT.SimpleTy != MVT::v4f32)
5584
0
    return 0;
5585
0
  if ((Subtarget->hasVLX())) {
5586
0
    return fastEmitInst_r(X86::VGETEXPPSZ128r, &X86::VR128XRegClass, Op0);
5587
0
  }
5588
0
  return 0;
5589
0
}
5590
5591
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5592
0
  if (RetVT.SimpleTy != MVT::v8f32)
5593
0
    return 0;
5594
0
  if ((Subtarget->hasVLX())) {
5595
0
    return fastEmitInst_r(X86::VGETEXPPSZ256r, &X86::VR256XRegClass, Op0);
5596
0
  }
5597
0
  return 0;
5598
0
}
5599
5600
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5601
0
  if (RetVT.SimpleTy != MVT::v16f32)
5602
0
    return 0;
5603
0
  if ((Subtarget->hasAVX512())) {
5604
0
    return fastEmitInst_r(X86::VGETEXPPSZr, &X86::VR512RegClass, Op0);
5605
0
  }
5606
0
  return 0;
5607
0
}
5608
5609
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5610
0
  if (RetVT.SimpleTy != MVT::v2f64)
5611
0
    return 0;
5612
0
  if ((Subtarget->hasVLX())) {
5613
0
    return fastEmitInst_r(X86::VGETEXPPDZ128r, &X86::VR128XRegClass, Op0);
5614
0
  }
5615
0
  return 0;
5616
0
}
5617
5618
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5619
0
  if (RetVT.SimpleTy != MVT::v4f64)
5620
0
    return 0;
5621
0
  if ((Subtarget->hasVLX())) {
5622
0
    return fastEmitInst_r(X86::VGETEXPPDZ256r, &X86::VR256XRegClass, Op0);
5623
0
  }
5624
0
  return 0;
5625
0
}
5626
5627
0
unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5628
0
  if (RetVT.SimpleTy != MVT::v8f64)
5629
0
    return 0;
5630
0
  if ((Subtarget->hasAVX512())) {
5631
0
    return fastEmitInst_r(X86::VGETEXPPDZr, &X86::VR512RegClass, Op0);
5632
0
  }
5633
0
  return 0;
5634
0
}
5635
5636
0
unsigned fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, unsigned Op0) {
5637
0
  switch (VT.SimpleTy) {
5638
0
  case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0);
5639
0
  case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0);
5640
0
  case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0);
5641
0
  case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0);
5642
0
  case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0);
5643
0
  case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0);
5644
0
  case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0);
5645
0
  case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0);
5646
0
  case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0);
5647
0
  default: return 0;
5648
0
  }
5649
0
}
5650
5651
// FastEmit functions for X86ISD::FGETEXP_SAE.
5652
5653
0
unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
5654
0
  if (RetVT.SimpleTy != MVT::v32f16)
5655
0
    return 0;
5656
0
  if ((Subtarget->hasFP16())) {
5657
0
    return fastEmitInst_r(X86::VGETEXPPHZrb, &X86::VR512RegClass, Op0);
5658
0
  }
5659
0
  return 0;
5660
0
}
5661
5662
0
unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
5663
0
  if (RetVT.SimpleTy != MVT::v16f32)
5664
0
    return 0;
5665
0
  if ((Subtarget->hasAVX512())) {
5666
0
    return fastEmitInst_r(X86::VGETEXPPSZrb, &X86::VR512RegClass, Op0);
5667
0
  }
5668
0
  return 0;
5669
0
}
5670
5671
0
unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5672
0
  if (RetVT.SimpleTy != MVT::v8f64)
5673
0
    return 0;
5674
0
  if ((Subtarget->hasAVX512())) {
5675
0
    return fastEmitInst_r(X86::VGETEXPPDZrb, &X86::VR512RegClass, Op0);
5676
0
  }
5677
0
  return 0;
5678
0
}
5679
5680
0
unsigned fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
5681
0
  switch (VT.SimpleTy) {
5682
0
  case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0);
5683
0
  case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0);
5684
0
  case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0);
5685
0
  default: return 0;
5686
0
  }
5687
0
}
5688
5689
// FastEmit functions for X86ISD::FRCP.
5690
5691
0
unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0) {
5692
0
  if (RetVT.SimpleTy != MVT::f32)
5693
0
    return 0;
5694
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5695
0
    return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0);
5696
0
  }
5697
0
  return 0;
5698
0
}
5699
5700
0
unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5701
0
  if (RetVT.SimpleTy != MVT::v4f32)
5702
0
    return 0;
5703
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5704
0
    return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0);
5705
0
  }
5706
0
  if ((Subtarget->hasAVX())) {
5707
0
    return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0);
5708
0
  }
5709
0
  return 0;
5710
0
}
5711
5712
0
unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5713
0
  if (RetVT.SimpleTy != MVT::v8f32)
5714
0
    return 0;
5715
0
  if ((Subtarget->hasAVX())) {
5716
0
    return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0);
5717
0
  }
5718
0
  return 0;
5719
0
}
5720
5721
0
unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0) {
5722
0
  switch (VT.SimpleTy) {
5723
0
  case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0);
5724
0
  case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0);
5725
0
  case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0);
5726
0
  default: return 0;
5727
0
  }
5728
0
}
5729
5730
// FastEmit functions for X86ISD::FRSQRT.
5731
5732
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5733
0
  if (RetVT.SimpleTy != MVT::f32)
5734
0
    return 0;
5735
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5736
0
    return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0);
5737
0
  }
5738
0
  return 0;
5739
0
}
5740
5741
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5742
0
  if (RetVT.SimpleTy != MVT::v4f32)
5743
0
    return 0;
5744
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5745
0
    return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0);
5746
0
  }
5747
0
  if ((Subtarget->hasAVX())) {
5748
0
    return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0);
5749
0
  }
5750
0
  return 0;
5751
0
}
5752
5753
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5754
0
  if (RetVT.SimpleTy != MVT::v8f32)
5755
0
    return 0;
5756
0
  if ((Subtarget->hasAVX())) {
5757
0
    return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0);
5758
0
  }
5759
0
  return 0;
5760
0
}
5761
5762
0
unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
5763
0
  switch (VT.SimpleTy) {
5764
0
  case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0);
5765
0
  case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0);
5766
0
  case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0);
5767
0
  default: return 0;
5768
0
  }
5769
0
}
5770
5771
// FastEmit functions for X86ISD::MMX_MOVD2W.
5772
5773
0
unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
5774
0
  if (RetVT.SimpleTy != MVT::i32)
5775
0
    return 0;
5776
0
  if ((Subtarget->hasMMX())) {
5777
0
    return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0);
5778
0
  }
5779
0
  return 0;
5780
0
}
5781
5782
0
unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0) {
5783
0
  switch (VT.SimpleTy) {
5784
0
  case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0);
5785
0
  default: return 0;
5786
0
  }
5787
0
}
5788
5789
// FastEmit functions for X86ISD::MMX_MOVW2D.
5790
5791
0
unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0) {
5792
0
  if (RetVT.SimpleTy != MVT::x86mmx)
5793
0
    return 0;
5794
0
  if ((Subtarget->hasMMX())) {
5795
0
    return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0);
5796
0
  }
5797
0
  return 0;
5798
0
}
5799
5800
0
unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0) {
5801
0
  switch (VT.SimpleTy) {
5802
0
  case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0);
5803
0
  default: return 0;
5804
0
  }
5805
0
}
5806
5807
// FastEmit functions for X86ISD::MOVDDUP.
5808
5809
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5810
0
  if (RetVT.SimpleTy != MVT::v2f64)
5811
0
    return 0;
5812
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
5813
0
    return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0);
5814
0
  }
5815
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5816
0
    return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0);
5817
0
  }
5818
0
  return 0;
5819
0
}
5820
5821
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5822
0
  if (RetVT.SimpleTy != MVT::v4f64)
5823
0
    return 0;
5824
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5825
0
    return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0);
5826
0
  }
5827
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5828
0
    return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0);
5829
0
  }
5830
0
  return 0;
5831
0
}
5832
5833
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
5834
0
  if (RetVT.SimpleTy != MVT::v8f64)
5835
0
    return 0;
5836
0
  if ((Subtarget->hasAVX512())) {
5837
0
    return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0);
5838
0
  }
5839
0
  return 0;
5840
0
}
5841
5842
0
unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
5843
0
  switch (VT.SimpleTy) {
5844
0
  case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0);
5845
0
  case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0);
5846
0
  case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0);
5847
0
  default: return 0;
5848
0
  }
5849
0
}
5850
5851
// FastEmit functions for X86ISD::MOVDQ2Q.
5852
5853
0
unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
5854
0
  if (RetVT.SimpleTy != MVT::x86mmx)
5855
0
    return 0;
5856
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
5857
0
    return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0);
5858
0
  }
5859
0
  return 0;
5860
0
}
5861
5862
0
unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0) {
5863
0
  switch (VT.SimpleTy) {
5864
0
  case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0);
5865
0
  default: return 0;
5866
0
  }
5867
0
}
5868
5869
// FastEmit functions for X86ISD::MOVMSK.
5870
5871
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
5872
0
  if (RetVT.SimpleTy != MVT::i32)
5873
0
    return 0;
5874
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5875
0
    return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0);
5876
0
  }
5877
0
  if ((Subtarget->hasAVX())) {
5878
0
    return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0);
5879
0
  }
5880
0
  return 0;
5881
0
}
5882
5883
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0) {
5884
0
  if (RetVT.SimpleTy != MVT::i32)
5885
0
    return 0;
5886
0
  if ((Subtarget->hasAVX2())) {
5887
0
    return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0);
5888
0
  }
5889
0
  return 0;
5890
0
}
5891
5892
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
5893
0
  if (RetVT.SimpleTy != MVT::i32)
5894
0
    return 0;
5895
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5896
0
    return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0);
5897
0
  }
5898
0
  if ((Subtarget->hasAVX())) {
5899
0
    return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0);
5900
0
  }
5901
0
  return 0;
5902
0
}
5903
5904
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
5905
0
  if (RetVT.SimpleTy != MVT::i32)
5906
0
    return 0;
5907
0
  if ((Subtarget->hasAVX())) {
5908
0
    return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0);
5909
0
  }
5910
0
  return 0;
5911
0
}
5912
5913
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
5914
0
  if (RetVT.SimpleTy != MVT::i32)
5915
0
    return 0;
5916
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5917
0
    return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0);
5918
0
  }
5919
0
  if ((Subtarget->hasAVX())) {
5920
0
    return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0);
5921
0
  }
5922
0
  return 0;
5923
0
}
5924
5925
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
5926
0
  if (RetVT.SimpleTy != MVT::i32)
5927
0
    return 0;
5928
0
  if ((Subtarget->hasAVX())) {
5929
0
    return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0);
5930
0
  }
5931
0
  return 0;
5932
0
}
5933
5934
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5935
0
  if (RetVT.SimpleTy != MVT::i32)
5936
0
    return 0;
5937
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
5938
0
    return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0);
5939
0
  }
5940
0
  if ((Subtarget->hasAVX())) {
5941
0
    return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0);
5942
0
  }
5943
0
  return 0;
5944
0
}
5945
5946
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
5947
0
  if (RetVT.SimpleTy != MVT::i32)
5948
0
    return 0;
5949
0
  if ((Subtarget->hasAVX())) {
5950
0
    return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0);
5951
0
  }
5952
0
  return 0;
5953
0
}
5954
5955
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5956
0
  if (RetVT.SimpleTy != MVT::i32)
5957
0
    return 0;
5958
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5959
0
    return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0);
5960
0
  }
5961
0
  if ((Subtarget->hasAVX())) {
5962
0
    return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0);
5963
0
  }
5964
0
  return 0;
5965
0
}
5966
5967
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
5968
0
  if (RetVT.SimpleTy != MVT::i32)
5969
0
    return 0;
5970
0
  if ((Subtarget->hasAVX())) {
5971
0
    return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0);
5972
0
  }
5973
0
  return 0;
5974
0
}
5975
5976
0
unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0) {
5977
0
  switch (VT.SimpleTy) {
5978
0
  case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0);
5979
0
  case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0);
5980
0
  case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0);
5981
0
  case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0);
5982
0
  case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0);
5983
0
  case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0);
5984
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0);
5985
0
  case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0);
5986
0
  case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0);
5987
0
  case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0);
5988
0
  default: return 0;
5989
0
  }
5990
0
}
5991
5992
// FastEmit functions for X86ISD::MOVQ2DQ.
5993
5994
0
unsigned fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, unsigned Op0) {
5995
0
  if (RetVT.SimpleTy != MVT::v2i64)
5996
0
    return 0;
5997
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
5998
0
    return fastEmitInst_r(X86::MMX_MOVQ2DQrr, &X86::VR128RegClass, Op0);
5999
0
  }
6000
0
  return 0;
6001
0
}
6002
6003
0
unsigned fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, unsigned Op0) {
6004
0
  switch (VT.SimpleTy) {
6005
0
  case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0);
6006
0
  default: return 0;
6007
0
  }
6008
0
}
6009
6010
// FastEmit functions for X86ISD::MOVSHDUP.
6011
6012
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6013
0
  if (RetVT.SimpleTy != MVT::v4i32)
6014
0
    return 0;
6015
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6016
0
    return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0);
6017
0
  }
6018
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6019
0
    return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0);
6020
0
  }
6021
0
  return 0;
6022
0
}
6023
6024
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
6025
0
  if (RetVT.SimpleTy != MVT::v8i32)
6026
0
    return 0;
6027
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6028
0
    return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0);
6029
0
  }
6030
0
  return 0;
6031
0
}
6032
6033
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6034
0
  if (RetVT.SimpleTy != MVT::v4f32)
6035
0
    return 0;
6036
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6037
0
    return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0);
6038
0
  }
6039
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6040
0
    return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0);
6041
0
  }
6042
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6043
0
    return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0);
6044
0
  }
6045
0
  return 0;
6046
0
}
6047
6048
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6049
0
  if (RetVT.SimpleTy != MVT::v8f32)
6050
0
    return 0;
6051
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6052
0
    return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0);
6053
0
  }
6054
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6055
0
    return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0);
6056
0
  }
6057
0
  return 0;
6058
0
}
6059
6060
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6061
0
  if (RetVT.SimpleTy != MVT::v16f32)
6062
0
    return 0;
6063
0
  if ((Subtarget->hasAVX512())) {
6064
0
    return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0);
6065
0
  }
6066
0
  return 0;
6067
0
}
6068
6069
0
unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
6070
0
  switch (VT.SimpleTy) {
6071
0
  case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0);
6072
0
  case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0);
6073
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0);
6074
0
  case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0);
6075
0
  case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0);
6076
0
  default: return 0;
6077
0
  }
6078
0
}
6079
6080
// FastEmit functions for X86ISD::MOVSLDUP.
6081
6082
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6083
0
  if (RetVT.SimpleTy != MVT::v4i32)
6084
0
    return 0;
6085
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6086
0
    return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0);
6087
0
  }
6088
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6089
0
    return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0);
6090
0
  }
6091
0
  return 0;
6092
0
}
6093
6094
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
6095
0
  if (RetVT.SimpleTy != MVT::v8i32)
6096
0
    return 0;
6097
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6098
0
    return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0);
6099
0
  }
6100
0
  return 0;
6101
0
}
6102
6103
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6104
0
  if (RetVT.SimpleTy != MVT::v4f32)
6105
0
    return 0;
6106
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6107
0
    return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0);
6108
0
  }
6109
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
6110
0
    return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0);
6111
0
  }
6112
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6113
0
    return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0);
6114
0
  }
6115
0
  return 0;
6116
0
}
6117
6118
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6119
0
  if (RetVT.SimpleTy != MVT::v8f32)
6120
0
    return 0;
6121
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6122
0
    return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0);
6123
0
  }
6124
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6125
0
    return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0);
6126
0
  }
6127
0
  return 0;
6128
0
}
6129
6130
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6131
0
  if (RetVT.SimpleTy != MVT::v16f32)
6132
0
    return 0;
6133
0
  if ((Subtarget->hasAVX512())) {
6134
0
    return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0);
6135
0
  }
6136
0
  return 0;
6137
0
}
6138
6139
0
unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
6140
0
  switch (VT.SimpleTy) {
6141
0
  case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0);
6142
0
  case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0);
6143
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0);
6144
0
  case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0);
6145
0
  case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0);
6146
0
  default: return 0;
6147
0
  }
6148
0
}
6149
6150
// FastEmit functions for X86ISD::NT_BRIND.
6151
6152
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) {
6153
0
  if (RetVT.SimpleTy != MVT::isVoid)
6154
0
    return 0;
6155
0
  if ((!Subtarget->is64Bit())) {
6156
0
    return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0);
6157
0
  }
6158
0
  return 0;
6159
0
}
6160
6161
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
6162
0
  if (RetVT.SimpleTy != MVT::isVoid)
6163
0
    return 0;
6164
0
  if ((!Subtarget->is64Bit())) {
6165
0
    return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0);
6166
0
  }
6167
0
  return 0;
6168
0
}
6169
6170
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) {
6171
0
  if (RetVT.SimpleTy != MVT::isVoid)
6172
0
    return 0;
6173
0
  if ((Subtarget->is64Bit())) {
6174
0
    return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0);
6175
0
  }
6176
0
  return 0;
6177
0
}
6178
6179
0
unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
6180
0
  switch (VT.SimpleTy) {
6181
0
  case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0);
6182
0
  case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0);
6183
0
  case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0);
6184
0
  default: return 0;
6185
0
  }
6186
0
}
6187
6188
// FastEmit functions for X86ISD::NT_CALL.
6189
6190
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) {
6191
0
  if (RetVT.SimpleTy != MVT::isVoid)
6192
0
    return 0;
6193
0
  if ((!Subtarget->is64Bit())) {
6194
0
    return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0);
6195
0
  }
6196
0
  return 0;
6197
0
}
6198
6199
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
6200
0
  if (RetVT.SimpleTy != MVT::isVoid)
6201
0
    return 0;
6202
0
  if ((!Subtarget->is64Bit())) {
6203
0
    return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0);
6204
0
  }
6205
0
  return 0;
6206
0
}
6207
6208
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) {
6209
0
  if (RetVT.SimpleTy != MVT::isVoid)
6210
0
    return 0;
6211
0
  if ((Subtarget->is64Bit())) {
6212
0
    return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0);
6213
0
  }
6214
0
  return 0;
6215
0
}
6216
6217
0
unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
6218
0
  switch (VT.SimpleTy) {
6219
0
  case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0);
6220
0
  case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0);
6221
0
  case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0);
6222
0
  default: return 0;
6223
0
  }
6224
0
}
6225
6226
// FastEmit functions for X86ISD::PHMINPOS.
6227
6228
0
unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6229
0
  if (RetVT.SimpleTy != MVT::v8i16)
6230
0
    return 0;
6231
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
6232
0
    return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0);
6233
0
  }
6234
0
  if ((Subtarget->hasAVX())) {
6235
0
    return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0);
6236
0
  }
6237
0
  return 0;
6238
0
}
6239
6240
0
unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0) {
6241
0
  switch (VT.SimpleTy) {
6242
0
  case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0);
6243
0
  default: return 0;
6244
0
  }
6245
0
}
6246
6247
// FastEmit functions for X86ISD::PROBED_ALLOCA.
6248
6249
0
unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
6250
0
  if (RetVT.SimpleTy != MVT::i32)
6251
0
    return 0;
6252
0
  if ((!Subtarget->isTarget64BitLP64())) {
6253
0
    return fastEmitInst_r(X86::PROBED_ALLOCA_32, &X86::GR32RegClass, Op0);
6254
0
  }
6255
0
  return 0;
6256
0
}
6257
6258
0
unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
6259
0
  if (RetVT.SimpleTy != MVT::i64)
6260
0
    return 0;
6261
0
  if ((Subtarget->is64Bit())) {
6262
0
    return fastEmitInst_r(X86::PROBED_ALLOCA_64, &X86::GR64RegClass, Op0);
6263
0
  }
6264
0
  return 0;
6265
0
}
6266
6267
0
unsigned fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
6268
0
  switch (VT.SimpleTy) {
6269
0
  case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0);
6270
0
  case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
6271
0
  default: return 0;
6272
0
  }
6273
0
}
6274
6275
// FastEmit functions for X86ISD::RCP14.
6276
6277
0
unsigned fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6278
0
  if (RetVT.SimpleTy != MVT::v8f16)
6279
0
    return 0;
6280
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6281
0
    return fastEmitInst_r(X86::VRCPPHZ128r, &X86::VR128XRegClass, Op0);
6282
0
  }
6283
0
  return 0;
6284
0
}
6285
6286
0
unsigned fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6287
0
  if (RetVT.SimpleTy != MVT::v16f16)
6288
0
    return 0;
6289
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6290
0
    return fastEmitInst_r(X86::VRCPPHZ256r, &X86::VR256XRegClass, Op0);
6291
0
  }
6292
0
  return 0;
6293
0
}
6294
6295
0
unsigned fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6296
0
  if (RetVT.SimpleTy != MVT::v32f16)
6297
0
    return 0;
6298
0
  if ((Subtarget->hasFP16())) {
6299
0
    return fastEmitInst_r(X86::VRCPPHZr, &X86::VR512RegClass, Op0);
6300
0
  }
6301
0
  return 0;
6302
0
}
6303
6304
0
unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6305
0
  if (RetVT.SimpleTy != MVT::v4f32)
6306
0
    return 0;
6307
0
  if ((Subtarget->hasVLX())) {
6308
0
    return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0);
6309
0
  }
6310
0
  return 0;
6311
0
}
6312
6313
0
unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6314
0
  if (RetVT.SimpleTy != MVT::v8f32)
6315
0
    return 0;
6316
0
  if ((Subtarget->hasVLX())) {
6317
0
    return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0);
6318
0
  }
6319
0
  return 0;
6320
0
}
6321
6322
0
unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6323
0
  if (RetVT.SimpleTy != MVT::v16f32)
6324
0
    return 0;
6325
0
  if ((Subtarget->hasAVX512())) {
6326
0
    return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0);
6327
0
  }
6328
0
  return 0;
6329
0
}
6330
6331
0
unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6332
0
  if (RetVT.SimpleTy != MVT::v2f64)
6333
0
    return 0;
6334
0
  if ((Subtarget->hasVLX())) {
6335
0
    return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0);
6336
0
  }
6337
0
  return 0;
6338
0
}
6339
6340
0
unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6341
0
  if (RetVT.SimpleTy != MVT::v4f64)
6342
0
    return 0;
6343
0
  if ((Subtarget->hasVLX())) {
6344
0
    return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0);
6345
0
  }
6346
0
  return 0;
6347
0
}
6348
6349
0
unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6350
0
  if (RetVT.SimpleTy != MVT::v8f64)
6351
0
    return 0;
6352
0
  if ((Subtarget->hasAVX512())) {
6353
0
    return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0);
6354
0
  }
6355
0
  return 0;
6356
0
}
6357
6358
0
unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0) {
6359
0
  switch (VT.SimpleTy) {
6360
0
  case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0);
6361
0
  case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0);
6362
0
  case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0);
6363
0
  case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0);
6364
0
  case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0);
6365
0
  case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0);
6366
0
  case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0);
6367
0
  case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0);
6368
0
  case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0);
6369
0
  default: return 0;
6370
0
  }
6371
0
}
6372
6373
// FastEmit functions for X86ISD::RCP28.
6374
6375
0
unsigned fastEmit_X86ISD_RCP28_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6376
0
  if (RetVT.SimpleTy != MVT::v16f32)
6377
0
    return 0;
6378
0
  if ((Subtarget->hasERI())) {
6379
0
    return fastEmitInst_r(X86::VRCP28PSZr, &X86::VR512RegClass, Op0);
6380
0
  }
6381
0
  return 0;
6382
0
}
6383
6384
0
unsigned fastEmit_X86ISD_RCP28_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6385
0
  if (RetVT.SimpleTy != MVT::v8f64)
6386
0
    return 0;
6387
0
  if ((Subtarget->hasERI())) {
6388
0
    return fastEmitInst_r(X86::VRCP28PDZr, &X86::VR512RegClass, Op0);
6389
0
  }
6390
0
  return 0;
6391
0
}
6392
6393
0
unsigned fastEmit_X86ISD_RCP28_r(MVT VT, MVT RetVT, unsigned Op0) {
6394
0
  switch (VT.SimpleTy) {
6395
0
  case MVT::v16f32: return fastEmit_X86ISD_RCP28_MVT_v16f32_r(RetVT, Op0);
6396
0
  case MVT::v8f64: return fastEmit_X86ISD_RCP28_MVT_v8f64_r(RetVT, Op0);
6397
0
  default: return 0;
6398
0
  }
6399
0
}
6400
6401
// FastEmit functions for X86ISD::RCP28_SAE.
6402
6403
0
unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6404
0
  if (RetVT.SimpleTy != MVT::v16f32)
6405
0
    return 0;
6406
0
  if ((Subtarget->hasERI())) {
6407
0
    return fastEmitInst_r(X86::VRCP28PSZrb, &X86::VR512RegClass, Op0);
6408
0
  }
6409
0
  return 0;
6410
0
}
6411
6412
0
unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6413
0
  if (RetVT.SimpleTy != MVT::v8f64)
6414
0
    return 0;
6415
0
  if ((Subtarget->hasERI())) {
6416
0
    return fastEmitInst_r(X86::VRCP28PDZrb, &X86::VR512RegClass, Op0);
6417
0
  }
6418
0
  return 0;
6419
0
}
6420
6421
0
unsigned fastEmit_X86ISD_RCP28_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
6422
0
  switch (VT.SimpleTy) {
6423
0
  case MVT::v16f32: return fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(RetVT, Op0);
6424
0
  case MVT::v8f64: return fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(RetVT, Op0);
6425
0
  default: return 0;
6426
0
  }
6427
0
}
6428
6429
// FastEmit functions for X86ISD::RSQRT14.
6430
6431
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6432
0
  if (RetVT.SimpleTy != MVT::v8f16)
6433
0
    return 0;
6434
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6435
0
    return fastEmitInst_r(X86::VRSQRTPHZ128r, &X86::VR128XRegClass, Op0);
6436
0
  }
6437
0
  return 0;
6438
0
}
6439
6440
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6441
0
  if (RetVT.SimpleTy != MVT::v16f16)
6442
0
    return 0;
6443
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6444
0
    return fastEmitInst_r(X86::VRSQRTPHZ256r, &X86::VR256XRegClass, Op0);
6445
0
  }
6446
0
  return 0;
6447
0
}
6448
6449
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6450
0
  if (RetVT.SimpleTy != MVT::v32f16)
6451
0
    return 0;
6452
0
  if ((Subtarget->hasFP16())) {
6453
0
    return fastEmitInst_r(X86::VRSQRTPHZr, &X86::VR512RegClass, Op0);
6454
0
  }
6455
0
  return 0;
6456
0
}
6457
6458
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6459
0
  if (RetVT.SimpleTy != MVT::v4f32)
6460
0
    return 0;
6461
0
  if ((Subtarget->hasVLX())) {
6462
0
    return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0);
6463
0
  }
6464
0
  return 0;
6465
0
}
6466
6467
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6468
0
  if (RetVT.SimpleTy != MVT::v8f32)
6469
0
    return 0;
6470
0
  if ((Subtarget->hasVLX())) {
6471
0
    return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0);
6472
0
  }
6473
0
  return 0;
6474
0
}
6475
6476
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6477
0
  if (RetVT.SimpleTy != MVT::v16f32)
6478
0
    return 0;
6479
0
  if ((Subtarget->hasAVX512())) {
6480
0
    return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0);
6481
0
  }
6482
0
  return 0;
6483
0
}
6484
6485
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6486
0
  if (RetVT.SimpleTy != MVT::v2f64)
6487
0
    return 0;
6488
0
  if ((Subtarget->hasVLX())) {
6489
0
    return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0);
6490
0
  }
6491
0
  return 0;
6492
0
}
6493
6494
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6495
0
  if (RetVT.SimpleTy != MVT::v4f64)
6496
0
    return 0;
6497
0
  if ((Subtarget->hasVLX())) {
6498
0
    return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0);
6499
0
  }
6500
0
  return 0;
6501
0
}
6502
6503
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6504
0
  if (RetVT.SimpleTy != MVT::v8f64)
6505
0
    return 0;
6506
0
  if ((Subtarget->hasAVX512())) {
6507
0
    return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0);
6508
0
  }
6509
0
  return 0;
6510
0
}
6511
6512
0
unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0) {
6513
0
  switch (VT.SimpleTy) {
6514
0
  case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0);
6515
0
  case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0);
6516
0
  case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0);
6517
0
  case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0);
6518
0
  case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0);
6519
0
  case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0);
6520
0
  case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0);
6521
0
  case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0);
6522
0
  case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0);
6523
0
  default: return 0;
6524
0
  }
6525
0
}
6526
6527
// FastEmit functions for X86ISD::RSQRT28.
6528
6529
0
unsigned fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6530
0
  if (RetVT.SimpleTy != MVT::v16f32)
6531
0
    return 0;
6532
0
  if ((Subtarget->hasERI())) {
6533
0
    return fastEmitInst_r(X86::VRSQRT28PSZr, &X86::VR512RegClass, Op0);
6534
0
  }
6535
0
  return 0;
6536
0
}
6537
6538
0
unsigned fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6539
0
  if (RetVT.SimpleTy != MVT::v8f64)
6540
0
    return 0;
6541
0
  if ((Subtarget->hasERI())) {
6542
0
    return fastEmitInst_r(X86::VRSQRT28PDZr, &X86::VR512RegClass, Op0);
6543
0
  }
6544
0
  return 0;
6545
0
}
6546
6547
0
unsigned fastEmit_X86ISD_RSQRT28_r(MVT VT, MVT RetVT, unsigned Op0) {
6548
0
  switch (VT.SimpleTy) {
6549
0
  case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(RetVT, Op0);
6550
0
  case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(RetVT, Op0);
6551
0
  default: return 0;
6552
0
  }
6553
0
}
6554
6555
// FastEmit functions for X86ISD::RSQRT28_SAE.
6556
6557
0
unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6558
0
  if (RetVT.SimpleTy != MVT::v16f32)
6559
0
    return 0;
6560
0
  if ((Subtarget->hasERI())) {
6561
0
    return fastEmitInst_r(X86::VRSQRT28PSZrb, &X86::VR512RegClass, Op0);
6562
0
  }
6563
0
  return 0;
6564
0
}
6565
6566
0
unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6567
0
  if (RetVT.SimpleTy != MVT::v8f64)
6568
0
    return 0;
6569
0
  if ((Subtarget->hasERI())) {
6570
0
    return fastEmitInst_r(X86::VRSQRT28PDZrb, &X86::VR512RegClass, Op0);
6571
0
  }
6572
0
  return 0;
6573
0
}
6574
6575
0
unsigned fastEmit_X86ISD_RSQRT28_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
6576
0
  switch (VT.SimpleTy) {
6577
0
  case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(RetVT, Op0);
6578
0
  case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(RetVT, Op0);
6579
0
  default: return 0;
6580
0
  }
6581
0
}
6582
6583
// FastEmit functions for X86ISD::SEG_ALLOCA.
6584
6585
0
unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) {
6586
0
  if (RetVT.SimpleTy != MVT::i32)
6587
0
    return 0;
6588
0
  if ((!Subtarget->isTarget64BitLP64())) {
6589
0
    return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0);
6590
0
  }
6591
0
  return 0;
6592
0
}
6593
6594
0
unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
6595
0
  if (RetVT.SimpleTy != MVT::i64)
6596
0
    return 0;
6597
0
  if ((Subtarget->is64Bit())) {
6598
0
    return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0);
6599
0
  }
6600
0
  return 0;
6601
0
}
6602
6603
0
unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
6604
0
  switch (VT.SimpleTy) {
6605
0
  case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0);
6606
0
  case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0);
6607
0
  default: return 0;
6608
0
  }
6609
0
}
6610
6611
// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
6612
6613
0
unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
6614
0
  if ((Subtarget->hasVLX())) {
6615
0
    return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0);
6616
0
  }
6617
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
6618
0
    return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0);
6619
0
  }
6620
0
  return 0;
6621
0
}
6622
6623
0
unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) {
6624
0
  if ((Subtarget->hasVLX())) {
6625
0
    return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0);
6626
0
  }
6627
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
6628
0
    return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0);
6629
0
  }
6630
0
  return 0;
6631
0
}
6632
6633
0
unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6634
0
switch (RetVT.SimpleTy) {
6635
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0);
6636
0
  case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0);
6637
0
  default: return 0;
6638
0
}
6639
0
}
6640
6641
0
unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
6642
0
  if (RetVT.SimpleTy != MVT::v16f32)
6643
0
    return 0;
6644
0
  if ((Subtarget->hasAVX512())) {
6645
0
    return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0);
6646
0
  }
6647
0
  return 0;
6648
0
}
6649
6650
0
unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) {
6651
0
  switch (VT.SimpleTy) {
6652
0
  case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0);
6653
0
  case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0);
6654
0
  default: return 0;
6655
0
  }
6656
0
}
6657
6658
// FastEmit functions for X86ISD::STRICT_CVTSI2P.
6659
6660
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
6661
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6662
0
    return fastEmitInst_r(X86::VCVTDQ2PHZ128rr, &X86::VR128XRegClass, Op0);
6663
0
  }
6664
0
  return 0;
6665
0
}
6666
6667
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
6668
0
  if ((Subtarget->hasVLX())) {
6669
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0);
6670
0
  }
6671
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6672
0
    return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0);
6673
0
  }
6674
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6675
0
    return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0);
6676
0
  }
6677
0
  return 0;
6678
0
}
6679
6680
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6681
0
switch (RetVT.SimpleTy) {
6682
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0);
6683
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0);
6684
0
  default: return 0;
6685
0
}
6686
0
}
6687
6688
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
6689
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6690
0
    return fastEmitInst_r(X86::VCVTQQ2PHZ128rr, &X86::VR128XRegClass, Op0);
6691
0
  }
6692
0
  return 0;
6693
0
}
6694
6695
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
6696
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6697
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0);
6698
0
  }
6699
0
  return 0;
6700
0
}
6701
6702
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6703
0
switch (RetVT.SimpleTy) {
6704
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0);
6705
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0);
6706
0
  default: return 0;
6707
0
}
6708
0
}
6709
6710
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
6711
0
  if (RetVT.SimpleTy != MVT::v8f16)
6712
0
    return 0;
6713
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6714
0
    return fastEmitInst_r(X86::VCVTQQ2PHZ256rr, &X86::VR128XRegClass, Op0);
6715
0
  }
6716
0
  return 0;
6717
0
}
6718
6719
0
unsigned fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
6720
0
  switch (VT.SimpleTy) {
6721
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0);
6722
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0);
6723
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0);
6724
0
  default: return 0;
6725
0
  }
6726
0
}
6727
6728
// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
6729
6730
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
6731
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6732
0
    return fastEmitInst_r(X86::VCVTTPH2WZ128rr, &X86::VR128XRegClass, Op0);
6733
0
  }
6734
0
  return 0;
6735
0
}
6736
6737
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
6738
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6739
0
    return fastEmitInst_r(X86::VCVTTPH2DQZ128rr, &X86::VR128XRegClass, Op0);
6740
0
  }
6741
0
  return 0;
6742
0
}
6743
6744
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
6745
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6746
0
    return fastEmitInst_r(X86::VCVTTPH2DQZ256rr, &X86::VR256XRegClass, Op0);
6747
0
  }
6748
0
  return 0;
6749
0
}
6750
6751
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
6752
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6753
0
    return fastEmitInst_r(X86::VCVTTPH2QQZ128rr, &X86::VR128XRegClass, Op0);
6754
0
  }
6755
0
  return 0;
6756
0
}
6757
6758
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
6759
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6760
0
    return fastEmitInst_r(X86::VCVTTPH2QQZ256rr, &X86::VR256XRegClass, Op0);
6761
0
  }
6762
0
  return 0;
6763
0
}
6764
6765
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
6766
0
  if ((Subtarget->hasFP16())) {
6767
0
    return fastEmitInst_r(X86::VCVTTPH2QQZrr, &X86::VR512RegClass, Op0);
6768
0
  }
6769
0
  return 0;
6770
0
}
6771
6772
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6773
0
switch (RetVT.SimpleTy) {
6774
0
  case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0);
6775
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0);
6776
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0);
6777
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0);
6778
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0);
6779
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0);
6780
0
  default: return 0;
6781
0
}
6782
0
}
6783
6784
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
6785
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6786
0
    return fastEmitInst_r(X86::VCVTTPH2WZ256rr, &X86::VR256XRegClass, Op0);
6787
0
  }
6788
0
  return 0;
6789
0
}
6790
6791
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
6792
0
  if ((Subtarget->hasFP16())) {
6793
0
    return fastEmitInst_r(X86::VCVTTPH2DQZrr, &X86::VR512RegClass, Op0);
6794
0
  }
6795
0
  return 0;
6796
0
}
6797
6798
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
6799
0
switch (RetVT.SimpleTy) {
6800
0
  case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0);
6801
0
  case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0);
6802
0
  default: return 0;
6803
0
}
6804
0
}
6805
6806
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
6807
0
  if (RetVT.SimpleTy != MVT::v32i16)
6808
0
    return 0;
6809
0
  if ((Subtarget->hasFP16())) {
6810
0
    return fastEmitInst_r(X86::VCVTTPH2WZrr, &X86::VR512RegClass, Op0);
6811
0
  }
6812
0
  return 0;
6813
0
}
6814
6815
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
6816
0
  if ((Subtarget->hasVLX())) {
6817
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0);
6818
0
  }
6819
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6820
0
    return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0);
6821
0
  }
6822
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6823
0
    return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0);
6824
0
  }
6825
0
  return 0;
6826
0
}
6827
6828
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
6829
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6830
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0);
6831
0
  }
6832
0
  return 0;
6833
0
}
6834
6835
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
6836
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6837
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0);
6838
0
  }
6839
0
  return 0;
6840
0
}
6841
6842
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6843
0
switch (RetVT.SimpleTy) {
6844
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0);
6845
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0);
6846
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0);
6847
0
  default: return 0;
6848
0
}
6849
0
}
6850
6851
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
6852
0
  if ((Subtarget->hasVLX())) {
6853
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0);
6854
0
  }
6855
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6856
0
    return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0);
6857
0
  }
6858
0
  return 0;
6859
0
}
6860
6861
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
6862
0
  if ((Subtarget->hasDQI())) {
6863
0
    return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0);
6864
0
  }
6865
0
  return 0;
6866
0
}
6867
6868
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
6869
0
switch (RetVT.SimpleTy) {
6870
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0);
6871
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0);
6872
0
  default: return 0;
6873
0
}
6874
0
}
6875
6876
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
6877
0
  if (RetVT.SimpleTy != MVT::v16i32)
6878
0
    return 0;
6879
0
  if ((Subtarget->hasAVX512())) {
6880
0
    return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0);
6881
0
  }
6882
0
  return 0;
6883
0
}
6884
6885
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
6886
0
  if ((Subtarget->hasVLX())) {
6887
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0);
6888
0
  }
6889
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6890
0
    return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0);
6891
0
  }
6892
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6893
0
    return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0);
6894
0
  }
6895
0
  return 0;
6896
0
}
6897
6898
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
6899
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6900
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0);
6901
0
  }
6902
0
  return 0;
6903
0
}
6904
6905
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6906
0
switch (RetVT.SimpleTy) {
6907
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0);
6908
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0);
6909
0
  default: return 0;
6910
0
}
6911
0
}
6912
6913
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
6914
0
  if ((Subtarget->hasVLX())) {
6915
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0);
6916
0
  }
6917
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6918
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0);
6919
0
  }
6920
0
  return 0;
6921
0
}
6922
6923
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
6924
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6925
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0);
6926
0
  }
6927
0
  return 0;
6928
0
}
6929
6930
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
6931
0
switch (RetVT.SimpleTy) {
6932
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0);
6933
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0);
6934
0
  default: return 0;
6935
0
}
6936
0
}
6937
6938
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
6939
0
  if ((Subtarget->hasAVX512())) {
6940
0
    return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0);
6941
0
  }
6942
0
  return 0;
6943
0
}
6944
6945
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
6946
0
  if ((Subtarget->hasDQI())) {
6947
0
    return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0);
6948
0
  }
6949
0
  return 0;
6950
0
}
6951
6952
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
6953
0
switch (RetVT.SimpleTy) {
6954
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0);
6955
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0);
6956
0
  default: return 0;
6957
0
}
6958
0
}
6959
6960
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) {
6961
0
  switch (VT.SimpleTy) {
6962
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0);
6963
0
  case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0);
6964
0
  case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0);
6965
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0);
6966
0
  case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0);
6967
0
  case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0);
6968
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0);
6969
0
  case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0);
6970
0
  case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0);
6971
0
  default: return 0;
6972
0
  }
6973
0
}
6974
6975
// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
6976
6977
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) {
6978
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6979
0
    return fastEmitInst_r(X86::VCVTTPH2UWZ128rr, &X86::VR128XRegClass, Op0);
6980
0
  }
6981
0
  return 0;
6982
0
}
6983
6984
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
6985
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6986
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZ128rr, &X86::VR128XRegClass, Op0);
6987
0
  }
6988
0
  return 0;
6989
0
}
6990
6991
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) {
6992
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
6993
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZ256rr, &X86::VR256XRegClass, Op0);
6994
0
  }
6995
0
  return 0;
6996
0
}
6997
6998
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
6999
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7000
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZ128rr, &X86::VR128XRegClass, Op0);
7001
0
  }
7002
0
  return 0;
7003
0
}
7004
7005
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) {
7006
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7007
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZ256rr, &X86::VR256XRegClass, Op0);
7008
0
  }
7009
0
  return 0;
7010
0
}
7011
7012
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) {
7013
0
  if ((Subtarget->hasFP16())) {
7014
0
    return fastEmitInst_r(X86::VCVTTPH2UQQZrr, &X86::VR512RegClass, Op0);
7015
0
  }
7016
0
  return 0;
7017
0
}
7018
7019
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7020
0
switch (RetVT.SimpleTy) {
7021
0
  case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0);
7022
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0);
7023
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0);
7024
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0);
7025
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0);
7026
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0);
7027
0
  default: return 0;
7028
0
}
7029
0
}
7030
7031
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) {
7032
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7033
0
    return fastEmitInst_r(X86::VCVTTPH2UWZ256rr, &X86::VR256XRegClass, Op0);
7034
0
  }
7035
0
  return 0;
7036
0
}
7037
7038
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) {
7039
0
  if ((Subtarget->hasFP16())) {
7040
0
    return fastEmitInst_r(X86::VCVTTPH2UDQZrr, &X86::VR512RegClass, Op0);
7041
0
  }
7042
0
  return 0;
7043
0
}
7044
7045
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
7046
0
switch (RetVT.SimpleTy) {
7047
0
  case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0);
7048
0
  case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0);
7049
0
  default: return 0;
7050
0
}
7051
0
}
7052
7053
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) {
7054
0
  if (RetVT.SimpleTy != MVT::v32i16)
7055
0
    return 0;
7056
0
  if ((Subtarget->hasFP16())) {
7057
0
    return fastEmitInst_r(X86::VCVTTPH2UWZrr, &X86::VR512RegClass, Op0);
7058
0
  }
7059
0
  return 0;
7060
0
}
7061
7062
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) {
7063
0
  if ((Subtarget->hasVLX())) {
7064
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0);
7065
0
  }
7066
0
  return 0;
7067
0
}
7068
7069
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
7070
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7071
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0);
7072
0
  }
7073
0
  return 0;
7074
0
}
7075
7076
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) {
7077
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7078
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0);
7079
0
  }
7080
0
  return 0;
7081
0
}
7082
7083
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7084
0
switch (RetVT.SimpleTy) {
7085
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0);
7086
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0);
7087
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0);
7088
0
  default: return 0;
7089
0
}
7090
0
}
7091
7092
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) {
7093
0
  if ((Subtarget->hasVLX())) {
7094
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0);
7095
0
  }
7096
0
  return 0;
7097
0
}
7098
7099
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) {
7100
0
  if ((Subtarget->hasDQI())) {
7101
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0);
7102
0
  }
7103
0
  return 0;
7104
0
}
7105
7106
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7107
0
switch (RetVT.SimpleTy) {
7108
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0);
7109
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0);
7110
0
  default: return 0;
7111
0
}
7112
0
}
7113
7114
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
7115
0
  if (RetVT.SimpleTy != MVT::v16i32)
7116
0
    return 0;
7117
0
  if ((Subtarget->hasAVX512())) {
7118
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0);
7119
0
  }
7120
0
  return 0;
7121
0
}
7122
7123
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
7124
0
  if ((Subtarget->hasVLX())) {
7125
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0);
7126
0
  }
7127
0
  return 0;
7128
0
}
7129
7130
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) {
7131
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7132
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0);
7133
0
  }
7134
0
  return 0;
7135
0
}
7136
7137
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7138
0
switch (RetVT.SimpleTy) {
7139
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0);
7140
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0);
7141
0
  default: return 0;
7142
0
}
7143
0
}
7144
7145
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) {
7146
0
  if ((Subtarget->hasVLX())) {
7147
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0);
7148
0
  }
7149
0
  return 0;
7150
0
}
7151
7152
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) {
7153
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7154
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0);
7155
0
  }
7156
0
  return 0;
7157
0
}
7158
7159
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
7160
0
switch (RetVT.SimpleTy) {
7161
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0);
7162
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0);
7163
0
  default: return 0;
7164
0
}
7165
0
}
7166
7167
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) {
7168
0
  if ((Subtarget->hasAVX512())) {
7169
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0);
7170
0
  }
7171
0
  return 0;
7172
0
}
7173
7174
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) {
7175
0
  if ((Subtarget->hasDQI())) {
7176
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0);
7177
0
  }
7178
0
  return 0;
7179
0
}
7180
7181
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
7182
0
switch (RetVT.SimpleTy) {
7183
0
  case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0);
7184
0
  case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0);
7185
0
  default: return 0;
7186
0
}
7187
0
}
7188
7189
0
unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) {
7190
0
  switch (VT.SimpleTy) {
7191
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0);
7192
0
  case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0);
7193
0
  case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0);
7194
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0);
7195
0
  case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0);
7196
0
  case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0);
7197
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0);
7198
0
  case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0);
7199
0
  case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0);
7200
0
  default: return 0;
7201
0
  }
7202
0
}
7203
7204
// FastEmit functions for X86ISD::STRICT_CVTUI2P.
7205
7206
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
7207
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7208
0
    return fastEmitInst_r(X86::VCVTUDQ2PHZ128rr, &X86::VR128XRegClass, Op0);
7209
0
  }
7210
0
  return 0;
7211
0
}
7212
7213
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
7214
0
  if ((Subtarget->hasVLX())) {
7215
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0);
7216
0
  }
7217
0
  return 0;
7218
0
}
7219
7220
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
7221
0
switch (RetVT.SimpleTy) {
7222
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0);
7223
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0);
7224
0
  default: return 0;
7225
0
}
7226
0
}
7227
7228
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
7229
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7230
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZ128rr, &X86::VR128XRegClass, Op0);
7231
0
  }
7232
0
  return 0;
7233
0
}
7234
7235
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
7236
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
7237
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0);
7238
0
  }
7239
0
  return 0;
7240
0
}
7241
7242
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
7243
0
switch (RetVT.SimpleTy) {
7244
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0);
7245
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0);
7246
0
  default: return 0;
7247
0
}
7248
0
}
7249
7250
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
7251
0
  if (RetVT.SimpleTy != MVT::v8f16)
7252
0
    return 0;
7253
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7254
0
    return fastEmitInst_r(X86::VCVTUQQ2PHZ256rr, &X86::VR128XRegClass, Op0);
7255
0
  }
7256
0
  return 0;
7257
0
}
7258
7259
0
unsigned fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) {
7260
0
  switch (VT.SimpleTy) {
7261
0
  case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0);
7262
0
  case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0);
7263
0
  case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0);
7264
0
  default: return 0;
7265
0
  }
7266
0
}
7267
7268
// FastEmit functions for X86ISD::STRICT_VFPEXT.
7269
7270
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
7271
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7272
0
    return fastEmitInst_r(X86::VCVTPH2PSXZ128rr, &X86::VR128XRegClass, Op0);
7273
0
  }
7274
0
  return 0;
7275
0
}
7276
7277
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
7278
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7279
0
    return fastEmitInst_r(X86::VCVTPH2PDZ128rr, &X86::VR128XRegClass, Op0);
7280
0
  }
7281
0
  return 0;
7282
0
}
7283
7284
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) {
7285
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7286
0
    return fastEmitInst_r(X86::VCVTPH2PDZ256rr, &X86::VR256XRegClass, Op0);
7287
0
  }
7288
0
  return 0;
7289
0
}
7290
7291
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7292
0
switch (RetVT.SimpleTy) {
7293
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
7294
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
7295
0
  case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
7296
0
  default: return 0;
7297
0
}
7298
0
}
7299
7300
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7301
0
  if (RetVT.SimpleTy != MVT::v2f64)
7302
0
    return 0;
7303
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7304
0
    return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0);
7305
0
  }
7306
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7307
0
    return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0);
7308
0
  }
7309
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7310
0
    return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0);
7311
0
  }
7312
0
  return 0;
7313
0
}
7314
7315
0
unsigned fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) {
7316
0
  switch (VT.SimpleTy) {
7317
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0);
7318
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0);
7319
0
  default: return 0;
7320
0
  }
7321
0
}
7322
7323
// FastEmit functions for X86ISD::STRICT_VFPROUND.
7324
7325
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7326
0
  if (RetVT.SimpleTy != MVT::v8f16)
7327
0
    return 0;
7328
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7329
0
    return fastEmitInst_r(X86::VCVTPS2PHXZ128rr, &X86::VR128XRegClass, Op0);
7330
0
  }
7331
0
  return 0;
7332
0
}
7333
7334
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7335
0
  if (RetVT.SimpleTy != MVT::v8f16)
7336
0
    return 0;
7337
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7338
0
    return fastEmitInst_r(X86::VCVTPS2PHXZ256rr, &X86::VR128XRegClass, Op0);
7339
0
  }
7340
0
  return 0;
7341
0
}
7342
7343
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
7344
0
  if (RetVT.SimpleTy != MVT::v16f16)
7345
0
    return 0;
7346
0
  if ((Subtarget->hasFP16())) {
7347
0
    return fastEmitInst_r(X86::VCVTPS2PHXZrr, &X86::VR256XRegClass, Op0);
7348
0
  }
7349
0
  return 0;
7350
0
}
7351
7352
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
7353
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7354
0
    return fastEmitInst_r(X86::VCVTPD2PHZ128rr, &X86::VR128XRegClass, Op0);
7355
0
  }
7356
0
  return 0;
7357
0
}
7358
7359
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
7360
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7361
0
    return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0);
7362
0
  }
7363
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7364
0
    return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0);
7365
0
  }
7366
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7367
0
    return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0);
7368
0
  }
7369
0
  return 0;
7370
0
}
7371
7372
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7373
0
switch (RetVT.SimpleTy) {
7374
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
7375
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
7376
0
  default: return 0;
7377
0
}
7378
0
}
7379
7380
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) {
7381
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7382
0
    return fastEmitInst_r(X86::VCVTPD2PHZ256rr, &X86::VR128XRegClass, Op0);
7383
0
  }
7384
0
  return 0;
7385
0
}
7386
7387
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) {
7388
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7389
0
    return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0);
7390
0
  }
7391
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7392
0
    return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0);
7393
0
  }
7394
0
  return 0;
7395
0
}
7396
7397
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
7398
0
switch (RetVT.SimpleTy) {
7399
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
7400
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
7401
0
  default: return 0;
7402
0
}
7403
0
}
7404
7405
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) {
7406
0
  if ((Subtarget->hasFP16())) {
7407
0
    return fastEmitInst_r(X86::VCVTPD2PHZrr, &X86::VR128XRegClass, Op0);
7408
0
  }
7409
0
  return 0;
7410
0
}
7411
7412
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) {
7413
0
  if ((Subtarget->hasAVX512())) {
7414
0
    return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0);
7415
0
  }
7416
0
  return 0;
7417
0
}
7418
7419
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
7420
0
switch (RetVT.SimpleTy) {
7421
0
  case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
7422
0
  case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
7423
0
  default: return 0;
7424
0
}
7425
0
}
7426
7427
0
unsigned fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
7428
0
  switch (VT.SimpleTy) {
7429
0
  case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0);
7430
0
  case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0);
7431
0
  case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0);
7432
0
  case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0);
7433
0
  case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0);
7434
0
  case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0);
7435
0
  default: return 0;
7436
0
  }
7437
0
}
7438
7439
// FastEmit functions for X86ISD::VBROADCAST.
7440
7441
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0) {
7442
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7443
0
    return fastEmitInst_r(X86::VPBROADCASTDrZ128rr, &X86::VR128XRegClass, Op0);
7444
0
  }
7445
0
  return 0;
7446
0
}
7447
7448
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0) {
7449
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7450
0
    return fastEmitInst_r(X86::VPBROADCASTDrZ256rr, &X86::VR256XRegClass, Op0);
7451
0
  }
7452
0
  return 0;
7453
0
}
7454
7455
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0) {
7456
0
  if ((Subtarget->hasAVX512())) {
7457
0
    return fastEmitInst_r(X86::VPBROADCASTDrZrr, &X86::VR512RegClass, Op0);
7458
0
  }
7459
0
  return 0;
7460
0
}
7461
7462
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
7463
0
switch (RetVT.SimpleTy) {
7464
0
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0);
7465
0
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0);
7466
0
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0);
7467
0
  default: return 0;
7468
0
}
7469
0
}
7470
7471
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0) {
7472
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7473
0
    return fastEmitInst_r(X86::VPBROADCASTQrZ128rr, &X86::VR128XRegClass, Op0);
7474
0
  }
7475
0
  return 0;
7476
0
}
7477
7478
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0) {
7479
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7480
0
    return fastEmitInst_r(X86::VPBROADCASTQrZ256rr, &X86::VR256XRegClass, Op0);
7481
0
  }
7482
0
  return 0;
7483
0
}
7484
7485
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0) {
7486
0
  if ((Subtarget->hasAVX512())) {
7487
0
    return fastEmitInst_r(X86::VPBROADCASTQrZrr, &X86::VR512RegClass, Op0);
7488
0
  }
7489
0
  return 0;
7490
0
}
7491
7492
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0) {
7493
0
switch (RetVT.SimpleTy) {
7494
0
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0);
7495
0
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0);
7496
0
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0);
7497
0
  default: return 0;
7498
0
}
7499
0
}
7500
7501
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0) {
7502
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7503
0
    return fastEmitInst_r(X86::VPBROADCASTBZ128rr, &X86::VR128XRegClass, Op0);
7504
0
  }
7505
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7506
0
    return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0);
7507
0
  }
7508
0
  return 0;
7509
0
}
7510
7511
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0) {
7512
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7513
0
    return fastEmitInst_r(X86::VPBROADCASTBZ256rr, &X86::VR256XRegClass, Op0);
7514
0
  }
7515
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7516
0
    return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0);
7517
0
  }
7518
0
  return 0;
7519
0
}
7520
7521
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0) {
7522
0
  if ((Subtarget->hasBWI())) {
7523
0
    return fastEmitInst_r(X86::VPBROADCASTBZrr, &X86::VR512RegClass, Op0);
7524
0
  }
7525
0
  return 0;
7526
0
}
7527
7528
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
7529
0
switch (RetVT.SimpleTy) {
7530
0
  case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0);
7531
0
  case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0);
7532
0
  case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0);
7533
0
  default: return 0;
7534
0
}
7535
0
}
7536
7537
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0) {
7538
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7539
0
    return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0);
7540
0
  }
7541
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7542
0
    return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0);
7543
0
  }
7544
0
  return 0;
7545
0
}
7546
7547
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0) {
7548
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7549
0
    return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0);
7550
0
  }
7551
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7552
0
    return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0);
7553
0
  }
7554
0
  return 0;
7555
0
}
7556
7557
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0) {
7558
0
  if ((Subtarget->hasBWI())) {
7559
0
    return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0);
7560
0
  }
7561
0
  return 0;
7562
0
}
7563
7564
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
7565
0
switch (RetVT.SimpleTy) {
7566
0
  case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0);
7567
0
  case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0);
7568
0
  case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0);
7569
0
  default: return 0;
7570
0
}
7571
0
}
7572
7573
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0) {
7574
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7575
0
    return fastEmitInst_r(X86::VPBROADCASTDZ128rr, &X86::VR128XRegClass, Op0);
7576
0
  }
7577
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7578
0
    return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0);
7579
0
  }
7580
0
  return 0;
7581
0
}
7582
7583
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0) {
7584
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7585
0
    return fastEmitInst_r(X86::VPBROADCASTDZ256rr, &X86::VR256XRegClass, Op0);
7586
0
  }
7587
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7588
0
    return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0);
7589
0
  }
7590
0
  return 0;
7591
0
}
7592
7593
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0) {
7594
0
  if ((Subtarget->hasAVX512())) {
7595
0
    return fastEmitInst_r(X86::VPBROADCASTDZrr, &X86::VR512RegClass, Op0);
7596
0
  }
7597
0
  return 0;
7598
0
}
7599
7600
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
7601
0
switch (RetVT.SimpleTy) {
7602
0
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0);
7603
0
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0);
7604
0
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0);
7605
0
  default: return 0;
7606
0
}
7607
0
}
7608
7609
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0) {
7610
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7611
0
    return fastEmitInst_r(X86::VPBROADCASTQZ128rr, &X86::VR128XRegClass, Op0);
7612
0
  }
7613
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7614
0
    return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0);
7615
0
  }
7616
0
  return 0;
7617
0
}
7618
7619
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0) {
7620
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7621
0
    return fastEmitInst_r(X86::VPBROADCASTQZ256rr, &X86::VR256XRegClass, Op0);
7622
0
  }
7623
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7624
0
    return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0);
7625
0
  }
7626
0
  return 0;
7627
0
}
7628
7629
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0) {
7630
0
  if ((Subtarget->hasAVX512())) {
7631
0
    return fastEmitInst_r(X86::VPBROADCASTQZrr, &X86::VR512RegClass, Op0);
7632
0
  }
7633
0
  return 0;
7634
0
}
7635
7636
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
7637
0
switch (RetVT.SimpleTy) {
7638
0
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0);
7639
0
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0);
7640
0
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0);
7641
0
  default: return 0;
7642
0
}
7643
0
}
7644
7645
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(unsigned Op0) {
7646
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7647
0
    return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0);
7648
0
  }
7649
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7650
0
    return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0);
7651
0
  }
7652
0
  return 0;
7653
0
}
7654
7655
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(unsigned Op0) {
7656
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7657
0
    return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0);
7658
0
  }
7659
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7660
0
    return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0);
7661
0
  }
7662
0
  return 0;
7663
0
}
7664
7665
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(unsigned Op0) {
7666
0
  if ((Subtarget->hasBWI())) {
7667
0
    return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0);
7668
0
  }
7669
0
  return 0;
7670
0
}
7671
7672
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7673
0
switch (RetVT.SimpleTy) {
7674
0
  case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0);
7675
0
  case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0);
7676
0
  case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0);
7677
0
  default: return 0;
7678
0
}
7679
0
}
7680
7681
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(unsigned Op0) {
7682
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
7683
0
    return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0);
7684
0
  }
7685
0
  return 0;
7686
0
}
7687
7688
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(unsigned Op0) {
7689
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
7690
0
    return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0);
7691
0
  }
7692
0
  return 0;
7693
0
}
7694
7695
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(unsigned Op0) {
7696
0
  if ((Subtarget->hasBF16())) {
7697
0
    return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0);
7698
0
  }
7699
0
  return 0;
7700
0
}
7701
7702
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
7703
0
switch (RetVT.SimpleTy) {
7704
0
  case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v8bf16_r(Op0);
7705
0
  case MVT::v16bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v16bf16_r(Op0);
7706
0
  case MVT::v32bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_MVT_v32bf16_r(Op0);
7707
0
  default: return 0;
7708
0
}
7709
0
}
7710
7711
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0) {
7712
0
  if ((Subtarget->hasVLX())) {
7713
0
    return fastEmitInst_r(X86::VBROADCASTSSZ128rr, &X86::VR128XRegClass, Op0);
7714
0
  }
7715
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7716
0
    return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0);
7717
0
  }
7718
0
  return 0;
7719
0
}
7720
7721
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0) {
7722
0
  if ((Subtarget->hasVLX())) {
7723
0
    return fastEmitInst_r(X86::VBROADCASTSSZ256rr, &X86::VR256XRegClass, Op0);
7724
0
  }
7725
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7726
0
    return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0);
7727
0
  }
7728
0
  return 0;
7729
0
}
7730
7731
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0) {
7732
0
  if ((Subtarget->hasAVX512())) {
7733
0
    return fastEmitInst_r(X86::VBROADCASTSSZrr, &X86::VR512RegClass, Op0);
7734
0
  }
7735
0
  return 0;
7736
0
}
7737
7738
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7739
0
switch (RetVT.SimpleTy) {
7740
0
  case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0);
7741
0
  case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0);
7742
0
  case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0);
7743
0
  default: return 0;
7744
0
}
7745
0
}
7746
7747
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0) {
7748
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7749
0
    return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0);
7750
0
  }
7751
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7752
0
    return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0);
7753
0
  }
7754
0
  return 0;
7755
0
}
7756
7757
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0) {
7758
0
  if ((Subtarget->hasVLX())) {
7759
0
    return fastEmitInst_r(X86::VBROADCASTSDZ256rr, &X86::VR256XRegClass, Op0);
7760
0
  }
7761
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7762
0
    return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0);
7763
0
  }
7764
0
  return 0;
7765
0
}
7766
7767
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0) {
7768
0
  if ((Subtarget->hasAVX512())) {
7769
0
    return fastEmitInst_r(X86::VBROADCASTSDZrr, &X86::VR512RegClass, Op0);
7770
0
  }
7771
0
  return 0;
7772
0
}
7773
7774
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
7775
0
switch (RetVT.SimpleTy) {
7776
0
  case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0);
7777
0
  case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0);
7778
0
  case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0);
7779
0
  default: return 0;
7780
0
}
7781
0
}
7782
7783
0
unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
7784
0
  switch (VT.SimpleTy) {
7785
0
  case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0);
7786
0
  case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0);
7787
0
  case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0);
7788
0
  case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0);
7789
0
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0);
7790
0
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0);
7791
0
  case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0);
7792
0
  case MVT::v8bf16: return fastEmit_X86ISD_VBROADCAST_MVT_v8bf16_r(RetVT, Op0);
7793
0
  case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0);
7794
0
  case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0);
7795
0
  default: return 0;
7796
0
  }
7797
0
}
7798
7799
// FastEmit functions for X86ISD::VBROADCASTM.
7800
7801
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0) {
7802
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7803
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0);
7804
0
  }
7805
0
  return 0;
7806
0
}
7807
7808
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0) {
7809
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7810
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0);
7811
0
  }
7812
0
  return 0;
7813
0
}
7814
7815
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0) {
7816
0
  if ((Subtarget->hasCDI())) {
7817
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0);
7818
0
  }
7819
0
  return 0;
7820
0
}
7821
7822
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0) {
7823
0
switch (RetVT.SimpleTy) {
7824
0
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0);
7825
0
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0);
7826
0
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0);
7827
0
  default: return 0;
7828
0
}
7829
0
}
7830
7831
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0) {
7832
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7833
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0);
7834
0
  }
7835
0
  return 0;
7836
0
}
7837
7838
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0) {
7839
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
7840
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0);
7841
0
  }
7842
0
  return 0;
7843
0
}
7844
7845
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0) {
7846
0
  if ((Subtarget->hasCDI())) {
7847
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0);
7848
0
  }
7849
0
  return 0;
7850
0
}
7851
7852
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0) {
7853
0
switch (RetVT.SimpleTy) {
7854
0
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0);
7855
0
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0);
7856
0
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0);
7857
0
  default: return 0;
7858
0
}
7859
0
}
7860
7861
0
unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0) {
7862
0
  switch (VT.SimpleTy) {
7863
0
  case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0);
7864
0
  case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0);
7865
0
  default: return 0;
7866
0
  }
7867
0
}
7868
7869
// FastEmit functions for X86ISD::VFPEXT.
7870
7871
0
unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
7872
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7873
0
    return fastEmitInst_r(X86::VCVTPH2PSXZ128rr, &X86::VR128XRegClass, Op0);
7874
0
  }
7875
0
  return 0;
7876
0
}
7877
7878
0
unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
7879
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7880
0
    return fastEmitInst_r(X86::VCVTPH2PDZ128rr, &X86::VR128XRegClass, Op0);
7881
0
  }
7882
0
  return 0;
7883
0
}
7884
7885
0
unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) {
7886
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7887
0
    return fastEmitInst_r(X86::VCVTPH2PDZ256rr, &X86::VR256XRegClass, Op0);
7888
0
  }
7889
0
  return 0;
7890
0
}
7891
7892
0
unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7893
0
switch (RetVT.SimpleTy) {
7894
0
  case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0);
7895
0
  case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0);
7896
0
  case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0);
7897
0
  default: return 0;
7898
0
}
7899
0
}
7900
7901
0
unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7902
0
  if (RetVT.SimpleTy != MVT::v2f64)
7903
0
    return 0;
7904
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7905
0
    return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0);
7906
0
  }
7907
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7908
0
    return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0);
7909
0
  }
7910
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7911
0
    return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0);
7912
0
  }
7913
0
  return 0;
7914
0
}
7915
7916
0
unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) {
7917
0
  switch (VT.SimpleTy) {
7918
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0);
7919
0
  case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0);
7920
0
  default: return 0;
7921
0
  }
7922
0
}
7923
7924
// FastEmit functions for X86ISD::VFPEXT_SAE.
7925
7926
0
unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
7927
0
  if (RetVT.SimpleTy != MVT::v8f64)
7928
0
    return 0;
7929
0
  if ((Subtarget->hasFP16())) {
7930
0
    return fastEmitInst_r(X86::VCVTPH2PDZrrb, &X86::VR512RegClass, Op0);
7931
0
  }
7932
0
  return 0;
7933
0
}
7934
7935
0
unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) {
7936
0
  if (RetVT.SimpleTy != MVT::v16f32)
7937
0
    return 0;
7938
0
  if ((Subtarget->hasFP16())) {
7939
0
    return fastEmitInst_r(X86::VCVTPH2PSXZrrb, &X86::VR512RegClass, Op0);
7940
0
  }
7941
0
  return 0;
7942
0
}
7943
7944
0
unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7945
0
  if (RetVT.SimpleTy != MVT::v8f64)
7946
0
    return 0;
7947
0
  if ((Subtarget->hasAVX512())) {
7948
0
    return fastEmitInst_r(X86::VCVTPS2PDZrrb, &X86::VR512RegClass, Op0);
7949
0
  }
7950
0
  return 0;
7951
0
}
7952
7953
0
unsigned fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, unsigned Op0) {
7954
0
  switch (VT.SimpleTy) {
7955
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0);
7956
0
  case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0);
7957
0
  case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0);
7958
0
  default: return 0;
7959
0
  }
7960
0
}
7961
7962
// FastEmit functions for X86ISD::VFPROUND.
7963
7964
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
7965
0
  if (RetVT.SimpleTy != MVT::v8f16)
7966
0
    return 0;
7967
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7968
0
    return fastEmitInst_r(X86::VCVTPS2PHXZ128rr, &X86::VR128XRegClass, Op0);
7969
0
  }
7970
0
  return 0;
7971
0
}
7972
7973
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(unsigned Op0) {
7974
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
7975
0
    return fastEmitInst_r(X86::VCVTPS2PHXZ256rr, &X86::VR128XRegClass, Op0);
7976
0
  }
7977
0
  return 0;
7978
0
}
7979
7980
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(unsigned Op0) {
7981
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
7982
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Z256rr, &X86::VR128XRegClass, Op0);
7983
0
  }
7984
0
  if ((Subtarget->hasAVXNECONVERT())) {
7985
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Yrr, &X86::VR128RegClass, Op0);
7986
0
  }
7987
0
  return 0;
7988
0
}
7989
7990
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) {
7991
0
switch (RetVT.SimpleTy) {
7992
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8f16_r(Op0);
7993
0
  case MVT::v8bf16: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_MVT_v8bf16_r(Op0);
7994
0
  default: return 0;
7995
0
}
7996
0
}
7997
7998
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(unsigned Op0) {
7999
0
  if ((Subtarget->hasFP16())) {
8000
0
    return fastEmitInst_r(X86::VCVTPS2PHXZrr, &X86::VR256XRegClass, Op0);
8001
0
  }
8002
0
  return 0;
8003
0
}
8004
8005
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(unsigned Op0) {
8006
0
  if ((Subtarget->hasBF16())) {
8007
0
    return fastEmitInst_r(X86::VCVTNEPS2BF16Zrr, &X86::VR256XRegClass, Op0);
8008
0
  }
8009
0
  return 0;
8010
0
}
8011
8012
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) {
8013
0
switch (RetVT.SimpleTy) {
8014
0
  case MVT::v16f16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16f16_r(Op0);
8015
0
  case MVT::v16bf16: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_MVT_v16bf16_r(Op0);
8016
0
  default: return 0;
8017
0
}
8018
0
}
8019
8020
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
8021
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8022
0
    return fastEmitInst_r(X86::VCVTPD2PHZ128rr, &X86::VR128XRegClass, Op0);
8023
0
  }
8024
0
  return 0;
8025
0
}
8026
8027
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
8028
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8029
0
    return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0);
8030
0
  }
8031
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8032
0
    return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0);
8033
0
  }
8034
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8035
0
    return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0);
8036
0
  }
8037
0
  return 0;
8038
0
}
8039
8040
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
8041
0
switch (RetVT.SimpleTy) {
8042
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0);
8043
0
  case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0);
8044
0
  default: return 0;
8045
0
}
8046
0
}
8047
8048
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) {
8049
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
8050
0
    return fastEmitInst_r(X86::VCVTPD2PHZ256rr, &X86::VR128XRegClass, Op0);
8051
0
  }
8052
0
  return 0;
8053
0
}
8054
8055
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) {
8056
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8057
0
    return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0);
8058
0
  }
8059
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8060
0
    return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0);
8061
0
  }
8062
0
  return 0;
8063
0
}
8064
8065
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) {
8066
0
switch (RetVT.SimpleTy) {
8067
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0);
8068
0
  case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0);
8069
0
  default: return 0;
8070
0
}
8071
0
}
8072
8073
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) {
8074
0
  if ((Subtarget->hasFP16())) {
8075
0
    return fastEmitInst_r(X86::VCVTPD2PHZrr, &X86::VR128XRegClass, Op0);
8076
0
  }
8077
0
  return 0;
8078
0
}
8079
8080
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) {
8081
0
  if ((Subtarget->hasAVX512())) {
8082
0
    return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0);
8083
0
  }
8084
0
  return 0;
8085
0
}
8086
8087
0
unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) {
8088
0
switch (RetVT.SimpleTy) {
8089
0
  case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0);
8090
0
  case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0);
8091
0
  default: return 0;
8092
0
}
8093
0
}
8094
8095
0
unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
8096
0
  switch (VT.SimpleTy) {
8097
0
  case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0);
8098
0
  case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0);
8099
0
  case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0);
8100
0
  case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0);
8101
0
  case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0);
8102
0
  case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0);
8103
0
  default: return 0;
8104
0
  }
8105
0
}
8106
8107
// FastEmit functions for X86ISD::VTRUNC.
8108
8109
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8110
0
  if (RetVT.SimpleTy != MVT::v16i8)
8111
0
    return 0;
8112
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8113
0
    return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0);
8114
0
  }
8115
0
  return 0;
8116
0
}
8117
8118
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8119
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8120
0
    return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0);
8121
0
  }
8122
0
  return 0;
8123
0
}
8124
8125
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8126
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8127
0
    return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0);
8128
0
  }
8129
0
  return 0;
8130
0
}
8131
8132
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8133
0
switch (RetVT.SimpleTy) {
8134
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0);
8135
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0);
8136
0
  default: return 0;
8137
0
}
8138
0
}
8139
8140
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8141
0
  if (RetVT.SimpleTy != MVT::v16i8)
8142
0
    return 0;
8143
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8144
0
    return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0);
8145
0
  }
8146
0
  return 0;
8147
0
}
8148
8149
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8150
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8151
0
    return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0);
8152
0
  }
8153
0
  return 0;
8154
0
}
8155
8156
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8157
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8158
0
    return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0);
8159
0
  }
8160
0
  return 0;
8161
0
}
8162
8163
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8164
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8165
0
    return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0);
8166
0
  }
8167
0
  return 0;
8168
0
}
8169
8170
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8171
0
switch (RetVT.SimpleTy) {
8172
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0);
8173
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0);
8174
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0);
8175
0
  default: return 0;
8176
0
}
8177
0
}
8178
8179
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8180
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8181
0
    return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0);
8182
0
  }
8183
0
  return 0;
8184
0
}
8185
8186
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8187
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8188
0
    return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0);
8189
0
  }
8190
0
  return 0;
8191
0
}
8192
8193
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8194
0
switch (RetVT.SimpleTy) {
8195
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0);
8196
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0);
8197
0
  default: return 0;
8198
0
}
8199
0
}
8200
8201
0
unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8202
0
  if (RetVT.SimpleTy != MVT::v16i8)
8203
0
    return 0;
8204
0
  if ((Subtarget->hasAVX512())) {
8205
0
    return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0);
8206
0
  }
8207
0
  return 0;
8208
0
}
8209
8210
0
unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) {
8211
0
  switch (VT.SimpleTy) {
8212
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0);
8213
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0);
8214
0
  case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0);
8215
0
  case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0);
8216
0
  case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0);
8217
0
  case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0);
8218
0
  default: return 0;
8219
0
  }
8220
0
}
8221
8222
// FastEmit functions for X86ISD::VTRUNCS.
8223
8224
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8225
0
  if (RetVT.SimpleTy != MVT::v16i8)
8226
0
    return 0;
8227
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8228
0
    return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0);
8229
0
  }
8230
0
  return 0;
8231
0
}
8232
8233
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
8234
0
  if (RetVT.SimpleTy != MVT::v16i8)
8235
0
    return 0;
8236
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8237
0
    return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0);
8238
0
  }
8239
0
  return 0;
8240
0
}
8241
8242
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
8243
0
  if (RetVT.SimpleTy != MVT::v32i8)
8244
0
    return 0;
8245
0
  if ((Subtarget->hasBWI())) {
8246
0
    return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0);
8247
0
  }
8248
0
  return 0;
8249
0
}
8250
8251
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8252
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8253
0
    return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0);
8254
0
  }
8255
0
  return 0;
8256
0
}
8257
8258
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8259
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8260
0
    return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0);
8261
0
  }
8262
0
  return 0;
8263
0
}
8264
8265
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8266
0
switch (RetVT.SimpleTy) {
8267
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0);
8268
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0);
8269
0
  default: return 0;
8270
0
}
8271
0
}
8272
8273
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) {
8274
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8275
0
    return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0);
8276
0
  }
8277
0
  return 0;
8278
0
}
8279
8280
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) {
8281
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8282
0
    return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0);
8283
0
  }
8284
0
  return 0;
8285
0
}
8286
8287
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8288
0
switch (RetVT.SimpleTy) {
8289
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0);
8290
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0);
8291
0
  default: return 0;
8292
0
}
8293
0
}
8294
8295
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
8296
0
  if ((Subtarget->hasAVX512())) {
8297
0
    return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0);
8298
0
  }
8299
0
  return 0;
8300
0
}
8301
8302
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
8303
0
  if ((Subtarget->hasAVX512())) {
8304
0
    return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0);
8305
0
  }
8306
0
  return 0;
8307
0
}
8308
8309
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
8310
0
switch (RetVT.SimpleTy) {
8311
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0);
8312
0
  case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0);
8313
0
  default: return 0;
8314
0
}
8315
0
}
8316
8317
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8318
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8319
0
    return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0);
8320
0
  }
8321
0
  return 0;
8322
0
}
8323
8324
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8325
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8326
0
    return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0);
8327
0
  }
8328
0
  return 0;
8329
0
}
8330
8331
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8332
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8333
0
    return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0);
8334
0
  }
8335
0
  return 0;
8336
0
}
8337
8338
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8339
0
switch (RetVT.SimpleTy) {
8340
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0);
8341
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0);
8342
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0);
8343
0
  default: return 0;
8344
0
}
8345
0
}
8346
8347
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8348
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8349
0
    return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0);
8350
0
  }
8351
0
  return 0;
8352
0
}
8353
8354
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8355
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8356
0
    return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0);
8357
0
  }
8358
0
  return 0;
8359
0
}
8360
8361
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) {
8362
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8363
0
    return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0);
8364
0
  }
8365
0
  return 0;
8366
0
}
8367
8368
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8369
0
switch (RetVT.SimpleTy) {
8370
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0);
8371
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0);
8372
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0);
8373
0
  default: return 0;
8374
0
}
8375
0
}
8376
8377
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) {
8378
0
  if ((Subtarget->hasAVX512())) {
8379
0
    return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0);
8380
0
  }
8381
0
  return 0;
8382
0
}
8383
8384
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
8385
0
  if ((Subtarget->hasAVX512())) {
8386
0
    return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0);
8387
0
  }
8388
0
  return 0;
8389
0
}
8390
8391
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
8392
0
  if ((Subtarget->hasAVX512())) {
8393
0
    return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0);
8394
0
  }
8395
0
  return 0;
8396
0
}
8397
8398
0
unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8399
0
switch (RetVT.SimpleTy) {
8400
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0);
8401
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0);
8402
0
  case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0);
8403
0
  default: return 0;
8404
0
}
8405
0
}
8406
8407
0
unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0) {
8408
0
  switch (VT.SimpleTy) {
8409
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0);
8410
0
  case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0);
8411
0
  case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0);
8412
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0);
8413
0
  case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0);
8414
0
  case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0);
8415
0
  case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0);
8416
0
  case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0);
8417
0
  case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0);
8418
0
  default: return 0;
8419
0
  }
8420
0
}
8421
8422
// FastEmit functions for X86ISD::VTRUNCUS.
8423
8424
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
8425
0
  if (RetVT.SimpleTy != MVT::v16i8)
8426
0
    return 0;
8427
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8428
0
    return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0);
8429
0
  }
8430
0
  return 0;
8431
0
}
8432
8433
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0) {
8434
0
  if (RetVT.SimpleTy != MVT::v16i8)
8435
0
    return 0;
8436
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8437
0
    return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0);
8438
0
  }
8439
0
  return 0;
8440
0
}
8441
8442
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0) {
8443
0
  if (RetVT.SimpleTy != MVT::v32i8)
8444
0
    return 0;
8445
0
  if ((Subtarget->hasBWI())) {
8446
0
    return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0);
8447
0
  }
8448
0
  return 0;
8449
0
}
8450
8451
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
8452
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8453
0
    return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0);
8454
0
  }
8455
0
  return 0;
8456
0
}
8457
8458
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
8459
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8460
0
    return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0);
8461
0
  }
8462
0
  return 0;
8463
0
}
8464
8465
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
8466
0
switch (RetVT.SimpleTy) {
8467
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0);
8468
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0);
8469
0
  default: return 0;
8470
0
}
8471
0
}
8472
8473
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) {
8474
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8475
0
    return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0);
8476
0
  }
8477
0
  return 0;
8478
0
}
8479
8480
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) {
8481
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8482
0
    return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0);
8483
0
  }
8484
0
  return 0;
8485
0
}
8486
8487
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0) {
8488
0
switch (RetVT.SimpleTy) {
8489
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0);
8490
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0);
8491
0
  default: return 0;
8492
0
}
8493
0
}
8494
8495
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) {
8496
0
  if ((Subtarget->hasAVX512())) {
8497
0
    return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0);
8498
0
  }
8499
0
  return 0;
8500
0
}
8501
8502
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) {
8503
0
  if ((Subtarget->hasAVX512())) {
8504
0
    return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0);
8505
0
  }
8506
0
  return 0;
8507
0
}
8508
8509
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0) {
8510
0
switch (RetVT.SimpleTy) {
8511
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0);
8512
0
  case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0);
8513
0
  default: return 0;
8514
0
}
8515
0
}
8516
8517
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
8518
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8519
0
    return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0);
8520
0
  }
8521
0
  return 0;
8522
0
}
8523
8524
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
8525
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8526
0
    return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0);
8527
0
  }
8528
0
  return 0;
8529
0
}
8530
8531
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
8532
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8533
0
    return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0);
8534
0
  }
8535
0
  return 0;
8536
0
}
8537
8538
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8539
0
switch (RetVT.SimpleTy) {
8540
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0);
8541
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0);
8542
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0);
8543
0
  default: return 0;
8544
0
}
8545
0
}
8546
8547
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) {
8548
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8549
0
    return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0);
8550
0
  }
8551
0
  return 0;
8552
0
}
8553
8554
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) {
8555
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8556
0
    return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0);
8557
0
  }
8558
0
  return 0;
8559
0
}
8560
8561
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) {
8562
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8563
0
    return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0);
8564
0
  }
8565
0
  return 0;
8566
0
}
8567
8568
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0) {
8569
0
switch (RetVT.SimpleTy) {
8570
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0);
8571
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0);
8572
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0);
8573
0
  default: return 0;
8574
0
}
8575
0
}
8576
8577
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) {
8578
0
  if ((Subtarget->hasAVX512())) {
8579
0
    return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0);
8580
0
  }
8581
0
  return 0;
8582
0
}
8583
8584
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) {
8585
0
  if ((Subtarget->hasAVX512())) {
8586
0
    return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0);
8587
0
  }
8588
0
  return 0;
8589
0
}
8590
8591
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) {
8592
0
  if ((Subtarget->hasAVX512())) {
8593
0
    return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0);
8594
0
  }
8595
0
  return 0;
8596
0
}
8597
8598
0
unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0) {
8599
0
switch (RetVT.SimpleTy) {
8600
0
  case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0);
8601
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0);
8602
0
  case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0);
8603
0
  default: return 0;
8604
0
}
8605
0
}
8606
8607
0
unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0) {
8608
0
  switch (VT.SimpleTy) {
8609
0
  case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0);
8610
0
  case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0);
8611
0
  case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0);
8612
0
  case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0);
8613
0
  case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0);
8614
0
  case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0);
8615
0
  case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0);
8616
0
  case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0);
8617
0
  case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0);
8618
0
  default: return 0;
8619
0
  }
8620
0
}
8621
8622
// FastEmit functions for X86ISD::VZEXT_MOVL.
8623
8624
0
unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
8625
0
  if (RetVT.SimpleTy != MVT::v2i64)
8626
0
    return 0;
8627
0
  if ((Subtarget->hasAVX512())) {
8628
0
    return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0);
8629
0
  }
8630
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8631
0
    return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0);
8632
0
  }
8633
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8634
0
    return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0);
8635
0
  }
8636
0
  return 0;
8637
0
}
8638
8639
0
unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
8640
0
  if (RetVT.SimpleTy != MVT::v2f64)
8641
0
    return 0;
8642
0
  if ((Subtarget->hasAVX512())) {
8643
0
    return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0);
8644
0
  }
8645
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8646
0
    return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0);
8647
0
  }
8648
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8649
0
    return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0);
8650
0
  }
8651
0
  return 0;
8652
0
}
8653
8654
0
unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0) {
8655
0
  switch (VT.SimpleTy) {
8656
0
  case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0);
8657
0
  case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0);
8658
0
  default: return 0;
8659
0
  }
8660
0
}
8661
8662
// Top-level FastEmit function.
8663
8664
0
unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override {
8665
0
  switch (Opcode) {
8666
0
  case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
8667
0
  case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
8668
0
  case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
8669
0
  case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
8670
0
  case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
8671
0
  case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
8672
0
  case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
8673
0
  case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
8674
0
  case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0);
8675
0
  case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
8676
0
  case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
8677
0
  case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
8678
0
  case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
8679
0
  case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
8680
0
  case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
8681
0
  case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
8682
0
  case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0);
8683
0
  case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0);
8684
0
  case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0);
8685
0
  case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
8686
0
  case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
8687
0
  case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
8688
0
  case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
8689
0
  case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
8690
0
  case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
8691
0
  case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
8692
0
  case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
8693
0
  case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
8694
0
  case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
8695
0
  case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
8696
0
  case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
8697
0
  case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
8698
0
  case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0);
8699
0
  case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0);
8700
0
  case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0);
8701
0
  case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0);
8702
0
  case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0);
8703
0
  case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0);
8704
0
  case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0);
8705
0
  case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0);
8706
0
  case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0);
8707
0
  case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0);
8708
0
  case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0);
8709
0
  case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0);
8710
0
  case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0);
8711
0
  case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0);
8712
0
  case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0);
8713
0
  case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0);
8714
0
  case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0);
8715
0
  case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0);
8716
0
  case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0);
8717
0
  case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0);
8718
0
  case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0);
8719
0
  case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0);
8720
0
  case X86ISD::EXP2: return fastEmit_X86ISD_EXP2_r(VT, RetVT, Op0);
8721
0
  case X86ISD::EXP2_SAE: return fastEmit_X86ISD_EXP2_SAE_r(VT, RetVT, Op0);
8722
0
  case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0);
8723
0
  case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0);
8724
0
  case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0);
8725
0
  case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0);
8726
0
  case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0);
8727
0
  case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0);
8728
0
  case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0);
8729
0
  case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0);
8730
0
  case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0);
8731
0
  case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0);
8732
0
  case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0);
8733
0
  case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0);
8734
0
  case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0);
8735
0
  case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0);
8736
0
  case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0);
8737
0
  case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
8738
0
  case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0);
8739
0
  case X86ISD::RCP28: return fastEmit_X86ISD_RCP28_r(VT, RetVT, Op0);
8740
0
  case X86ISD::RCP28_SAE: return fastEmit_X86ISD_RCP28_SAE_r(VT, RetVT, Op0);
8741
0
  case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0);
8742
0
  case X86ISD::RSQRT28: return fastEmit_X86ISD_RSQRT28_r(VT, RetVT, Op0);
8743
0
  case X86ISD::RSQRT28_SAE: return fastEmit_X86ISD_RSQRT28_SAE_r(VT, RetVT, Op0);
8744
0
  case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0);
8745
0
  case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0);
8746
0
  case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0);
8747
0
  case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0);
8748
0
  case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0);
8749
0
  case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0);
8750
0
  case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0);
8751
0
  case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0);
8752
0
  case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0);
8753
0
  case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0);
8754
0
  case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0);
8755
0
  case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0);
8756
0
  case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0);
8757
0
  case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0);
8758
0
  case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0);
8759
0
  case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0);
8760
0
  case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0);
8761
0
  default: return 0;
8762
0
  }
8763
0
}
8764
8765
// FastEmit functions for ISD::ADD.
8766
8767
0
unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8768
0
  if (RetVT.SimpleTy != MVT::i8)
8769
0
    return 0;
8770
0
  if ((Subtarget->hasNDD())) {
8771
0
    return fastEmitInst_rr(X86::ADD8rr_ND, &X86::GR8RegClass, Op0, Op1);
8772
0
  }
8773
0
  if ((!Subtarget->hasNDD())) {
8774
0
    return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op1);
8775
0
  }
8776
0
  return 0;
8777
0
}
8778
8779
0
unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8780
0
  if (RetVT.SimpleTy != MVT::i16)
8781
0
    return 0;
8782
0
  if ((Subtarget->hasNDD())) {
8783
0
    return fastEmitInst_rr(X86::ADD16rr_ND, &X86::GR16RegClass, Op0, Op1);
8784
0
  }
8785
0
  if ((!Subtarget->hasNDD())) {
8786
0
    return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op1);
8787
0
  }
8788
0
  return 0;
8789
0
}
8790
8791
0
unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8792
0
  if (RetVT.SimpleTy != MVT::i32)
8793
0
    return 0;
8794
0
  if ((Subtarget->hasNDD())) {
8795
0
    return fastEmitInst_rr(X86::ADD32rr_ND, &X86::GR32RegClass, Op0, Op1);
8796
0
  }
8797
0
  if ((!Subtarget->hasNDD())) {
8798
0
    return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op1);
8799
0
  }
8800
0
  return 0;
8801
0
}
8802
8803
0
unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8804
0
  if (RetVT.SimpleTy != MVT::i64)
8805
0
    return 0;
8806
0
  if ((Subtarget->hasNDD())) {
8807
0
    return fastEmitInst_rr(X86::ADD64rr_ND, &X86::GR64RegClass, Op0, Op1);
8808
0
  }
8809
0
  if ((!Subtarget->hasNDD())) {
8810
0
    return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op1);
8811
0
  }
8812
0
  return 0;
8813
0
}
8814
8815
0
unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8816
0
  if (RetVT.SimpleTy != MVT::v16i8)
8817
0
    return 0;
8818
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8819
0
    return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op1);
8820
0
  }
8821
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8822
0
    return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op1);
8823
0
  }
8824
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8825
0
    return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op1);
8826
0
  }
8827
0
  return 0;
8828
0
}
8829
8830
0
unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8831
0
  if (RetVT.SimpleTy != MVT::v32i8)
8832
0
    return 0;
8833
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8834
0
    return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op1);
8835
0
  }
8836
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8837
0
    return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op1);
8838
0
  }
8839
0
  return 0;
8840
0
}
8841
8842
0
unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8843
0
  if (RetVT.SimpleTy != MVT::v64i8)
8844
0
    return 0;
8845
0
  if ((Subtarget->hasBWI())) {
8846
0
    return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op1);
8847
0
  }
8848
0
  return 0;
8849
0
}
8850
8851
0
unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8852
0
  if (RetVT.SimpleTy != MVT::v8i16)
8853
0
    return 0;
8854
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8855
0
    return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op1);
8856
0
  }
8857
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8858
0
    return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op1);
8859
0
  }
8860
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8861
0
    return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op1);
8862
0
  }
8863
0
  return 0;
8864
0
}
8865
8866
0
unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8867
0
  if (RetVT.SimpleTy != MVT::v16i16)
8868
0
    return 0;
8869
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8870
0
    return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op1);
8871
0
  }
8872
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8873
0
    return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op1);
8874
0
  }
8875
0
  return 0;
8876
0
}
8877
8878
0
unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8879
0
  if (RetVT.SimpleTy != MVT::v32i16)
8880
0
    return 0;
8881
0
  if ((Subtarget->hasBWI())) {
8882
0
    return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op1);
8883
0
  }
8884
0
  return 0;
8885
0
}
8886
8887
0
unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8888
0
  if (RetVT.SimpleTy != MVT::v4i32)
8889
0
    return 0;
8890
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8891
0
    return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op1);
8892
0
  }
8893
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8894
0
    return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op1);
8895
0
  }
8896
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8897
0
    return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op1);
8898
0
  }
8899
0
  return 0;
8900
0
}
8901
8902
0
unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8903
0
  if (RetVT.SimpleTy != MVT::v8i32)
8904
0
    return 0;
8905
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8906
0
    return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op1);
8907
0
  }
8908
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8909
0
    return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op1);
8910
0
  }
8911
0
  return 0;
8912
0
}
8913
8914
0
unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8915
0
  if (RetVT.SimpleTy != MVT::v16i32)
8916
0
    return 0;
8917
0
  if ((Subtarget->hasAVX512())) {
8918
0
    return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op1);
8919
0
  }
8920
0
  return 0;
8921
0
}
8922
8923
0
unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8924
0
  if (RetVT.SimpleTy != MVT::v2i64)
8925
0
    return 0;
8926
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8927
0
    return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
8928
0
  }
8929
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8930
0
    return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op1);
8931
0
  }
8932
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8933
0
    return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op1);
8934
0
  }
8935
0
  return 0;
8936
0
}
8937
8938
0
unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8939
0
  if (RetVT.SimpleTy != MVT::v4i64)
8940
0
    return 0;
8941
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8942
0
    return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
8943
0
  }
8944
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8945
0
    return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op1);
8946
0
  }
8947
0
  return 0;
8948
0
}
8949
8950
0
unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8951
0
  if (RetVT.SimpleTy != MVT::v8i64)
8952
0
    return 0;
8953
0
  if ((Subtarget->hasAVX512())) {
8954
0
    return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op1);
8955
0
  }
8956
0
  return 0;
8957
0
}
8958
8959
0
unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8960
0
  switch (VT.SimpleTy) {
8961
0
  case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1);
8962
0
  case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1);
8963
0
  case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
8964
0
  case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
8965
0
  case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
8966
0
  case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1);
8967
0
  case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1);
8968
0
  case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
8969
0
  case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1);
8970
0
  case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1);
8971
0
  case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
8972
0
  case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1);
8973
0
  case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1);
8974
0
  case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
8975
0
  case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1);
8976
0
  case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1);
8977
0
  default: return 0;
8978
0
  }
8979
0
}
8980
8981
// FastEmit functions for ISD::AND.
8982
8983
0
unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8984
0
  if (RetVT.SimpleTy != MVT::i8)
8985
0
    return 0;
8986
0
  if ((Subtarget->hasNDD())) {
8987
0
    return fastEmitInst_rr(X86::AND8rr_ND, &X86::GR8RegClass, Op0, Op1);
8988
0
  }
8989
0
  if ((!Subtarget->hasNDD())) {
8990
0
    return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op1);
8991
0
  }
8992
0
  return 0;
8993
0
}
8994
8995
0
unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8996
0
  if (RetVT.SimpleTy != MVT::i16)
8997
0
    return 0;
8998
0
  if ((Subtarget->hasNDD())) {
8999
0
    return fastEmitInst_rr(X86::AND16rr_ND, &X86::GR16RegClass, Op0, Op1);
9000
0
  }
9001
0
  if ((!Subtarget->hasNDD())) {
9002
0
    return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op1);
9003
0
  }
9004
0
  return 0;
9005
0
}
9006
9007
0
unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9008
0
  if (RetVT.SimpleTy != MVT::i32)
9009
0
    return 0;
9010
0
  if ((Subtarget->hasNDD())) {
9011
0
    return fastEmitInst_rr(X86::AND32rr_ND, &X86::GR32RegClass, Op0, Op1);
9012
0
  }
9013
0
  if ((!Subtarget->hasNDD())) {
9014
0
    return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op1);
9015
0
  }
9016
0
  return 0;
9017
0
}
9018
9019
0
unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9020
0
  if (RetVT.SimpleTy != MVT::i64)
9021
0
    return 0;
9022
0
  if ((Subtarget->hasNDD())) {
9023
0
    return fastEmitInst_rr(X86::AND64rr_ND, &X86::GR64RegClass, Op0, Op1);
9024
0
  }
9025
0
  if ((!Subtarget->hasNDD())) {
9026
0
    return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op1);
9027
0
  }
9028
0
  return 0;
9029
0
}
9030
9031
0
unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9032
0
  if (RetVT.SimpleTy != MVT::v8i1)
9033
0
    return 0;
9034
0
  if ((Subtarget->hasDQI())) {
9035
0
    return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op1);
9036
0
  }
9037
0
  return 0;
9038
0
}
9039
9040
0
unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9041
0
  if (RetVT.SimpleTy != MVT::v16i1)
9042
0
    return 0;
9043
0
  if ((Subtarget->hasAVX512())) {
9044
0
    return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op1);
9045
0
  }
9046
0
  return 0;
9047
0
}
9048
9049
0
unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9050
0
  if (RetVT.SimpleTy != MVT::v32i1)
9051
0
    return 0;
9052
0
  if ((Subtarget->hasBWI())) {
9053
0
    return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op1);
9054
0
  }
9055
0
  return 0;
9056
0
}
9057
9058
0
unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9059
0
  if (RetVT.SimpleTy != MVT::v64i1)
9060
0
    return 0;
9061
0
  if ((Subtarget->hasBWI())) {
9062
0
    return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op1);
9063
0
  }
9064
0
  return 0;
9065
0
}
9066
9067
0
unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9068
0
  if (RetVT.SimpleTy != MVT::v16i8)
9069
0
    return 0;
9070
0
  if ((Subtarget->hasVLX())) {
9071
0
    return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
9072
0
  }
9073
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9074
0
    return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1);
9075
0
  }
9076
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9077
0
    return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1);
9078
0
  }
9079
0
  return 0;
9080
0
}
9081
9082
0
unsigned fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9083
0
  if (RetVT.SimpleTy != MVT::v32i8)
9084
0
    return 0;
9085
0
  if ((Subtarget->hasVLX())) {
9086
0
    return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
9087
0
  }
9088
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9089
0
    return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1);
9090
0
  }
9091
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9092
0
    return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1);
9093
0
  }
9094
0
  return 0;
9095
0
}
9096
9097
0
unsigned fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9098
0
  if (RetVT.SimpleTy != MVT::v64i8)
9099
0
    return 0;
9100
0
  if ((Subtarget->hasAVX512())) {
9101
0
    return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1);
9102
0
  }
9103
0
  return 0;
9104
0
}
9105
9106
0
unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9107
0
  if (RetVT.SimpleTy != MVT::v8i16)
9108
0
    return 0;
9109
0
  if ((Subtarget->hasVLX())) {
9110
0
    return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
9111
0
  }
9112
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9113
0
    return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1);
9114
0
  }
9115
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9116
0
    return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1);
9117
0
  }
9118
0
  return 0;
9119
0
}
9120
9121
0
unsigned fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9122
0
  if (RetVT.SimpleTy != MVT::v16i16)
9123
0
    return 0;
9124
0
  if ((Subtarget->hasVLX())) {
9125
0
    return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
9126
0
  }
9127
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9128
0
    return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1);
9129
0
  }
9130
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9131
0
    return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1);
9132
0
  }
9133
0
  return 0;
9134
0
}
9135
9136
0
unsigned fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9137
0
  if (RetVT.SimpleTy != MVT::v32i16)
9138
0
    return 0;
9139
0
  if ((Subtarget->hasAVX512())) {
9140
0
    return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1);
9141
0
  }
9142
0
  return 0;
9143
0
}
9144
9145
0
unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9146
0
  if (RetVT.SimpleTy != MVT::v4i32)
9147
0
    return 0;
9148
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9149
0
    return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1);
9150
0
  }
9151
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9152
0
    return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1);
9153
0
  }
9154
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9155
0
    return fastEmitInst_rr(X86::VPANDDZ128rr, &X86::VR128XRegClass, Op0, Op1);
9156
0
  }
9157
0
  return 0;
9158
0
}
9159
9160
0
unsigned fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9161
0
  if (RetVT.SimpleTy != MVT::v8i32)
9162
0
    return 0;
9163
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9164
0
    return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1);
9165
0
  }
9166
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9167
0
    return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1);
9168
0
  }
9169
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9170
0
    return fastEmitInst_rr(X86::VPANDDZ256rr, &X86::VR256XRegClass, Op0, Op1);
9171
0
  }
9172
0
  return 0;
9173
0
}
9174
9175
0
unsigned fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9176
0
  if (RetVT.SimpleTy != MVT::v16i32)
9177
0
    return 0;
9178
0
  if ((Subtarget->hasAVX512())) {
9179
0
    return fastEmitInst_rr(X86::VPANDDZrr, &X86::VR512RegClass, Op0, Op1);
9180
0
  }
9181
0
  return 0;
9182
0
}
9183
9184
0
unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9185
0
  if (RetVT.SimpleTy != MVT::v2i64)
9186
0
    return 0;
9187
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9188
0
    return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
9189
0
  }
9190
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9191
0
    return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1);
9192
0
  }
9193
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9194
0
    return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1);
9195
0
  }
9196
0
  return 0;
9197
0
}
9198
9199
0
unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9200
0
  if (RetVT.SimpleTy != MVT::v4i64)
9201
0
    return 0;
9202
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
9203
0
    return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1);
9204
0
  }
9205
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9206
0
    return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
9207
0
  }
9208
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9209
0
    return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1);
9210
0
  }
9211
0
  return 0;
9212
0
}
9213
9214
0
unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9215
0
  if (RetVT.SimpleTy != MVT::v8i64)
9216
0
    return 0;
9217
0
  if ((Subtarget->hasAVX512())) {
9218
0
    return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1);
9219
0
  }
9220
0
  return 0;
9221
0
}
9222
9223
0
unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9224
0
  switch (VT.SimpleTy) {
9225
0
  case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1);
9226
0
  case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1);
9227
0
  case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
9228
0
  case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
9229
0
  case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1);
9230
0
  case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1);
9231
0
  case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1);
9232
0
  case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1);
9233
0
  case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
9234
0
  case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1);
9235
0
  case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1);
9236
0
  case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
9237
0
  case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1);
9238
0
  case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1);
9239
0
  case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
9240
0
  case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1);
9241
0
  case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1);
9242
0
  case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
9243
0
  case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1);
9244
0
  case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1);
9245
0
  default: return 0;
9246
0
  }
9247
0
}
9248
9249
// FastEmit functions for ISD::AVGCEILU.
9250
9251
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9252
0
  if (RetVT.SimpleTy != MVT::v16i8)
9253
0
    return 0;
9254
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9255
0
    return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op1);
9256
0
  }
9257
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9258
0
    return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op1);
9259
0
  }
9260
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9261
0
    return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op1);
9262
0
  }
9263
0
  return 0;
9264
0
}
9265
9266
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9267
0
  if (RetVT.SimpleTy != MVT::v32i8)
9268
0
    return 0;
9269
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9270
0
    return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op1);
9271
0
  }
9272
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9273
0
    return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op1);
9274
0
  }
9275
0
  return 0;
9276
0
}
9277
9278
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9279
0
  if (RetVT.SimpleTy != MVT::v64i8)
9280
0
    return 0;
9281
0
  if ((Subtarget->hasBWI())) {
9282
0
    return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op1);
9283
0
  }
9284
0
  return 0;
9285
0
}
9286
9287
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9288
0
  if (RetVT.SimpleTy != MVT::v8i16)
9289
0
    return 0;
9290
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9291
0
    return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op1);
9292
0
  }
9293
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9294
0
    return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op1);
9295
0
  }
9296
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9297
0
    return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op1);
9298
0
  }
9299
0
  return 0;
9300
0
}
9301
9302
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9303
0
  if (RetVT.SimpleTy != MVT::v16i16)
9304
0
    return 0;
9305
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9306
0
    return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op1);
9307
0
  }
9308
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9309
0
    return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op1);
9310
0
  }
9311
0
  return 0;
9312
0
}
9313
9314
0
unsigned fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9315
0
  if (RetVT.SimpleTy != MVT::v32i16)
9316
0
    return 0;
9317
0
  if ((Subtarget->hasBWI())) {
9318
0
    return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op1);
9319
0
  }
9320
0
  return 0;
9321
0
}
9322
9323
0
unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9324
0
  switch (VT.SimpleTy) {
9325
0
  case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
9326
0
  case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1);
9327
0
  case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1);
9328
0
  case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
9329
0
  case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1);
9330
0
  case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1);
9331
0
  default: return 0;
9332
0
  }
9333
0
}
9334
9335
// FastEmit functions for ISD::FADD.
9336
9337
0
unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9338
0
  if (RetVT.SimpleTy != MVT::f16)
9339
0
    return 0;
9340
0
  if ((Subtarget->hasFP16())) {
9341
0
    return fastEmitInst_rr(X86::VADDSHZrr, &X86::FR16XRegClass, Op0, Op1);
9342
0
  }
9343
0
  return 0;
9344
0
}
9345
9346
0
unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9347
0
  if (RetVT.SimpleTy != MVT::f32)
9348
0
    return 0;
9349
0
  if ((Subtarget->hasAVX512())) {
9350
0
    return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op1);
9351
0
  }
9352
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9353
0
    return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op1);
9354
0
  }
9355
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9356
0
    return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op1);
9357
0
  }
9358
0
  if ((!Subtarget->hasSSE1())) {
9359
0
    return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op1);
9360
0
  }
9361
0
  return 0;
9362
0
}
9363
9364
0
unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9365
0
  if (RetVT.SimpleTy != MVT::f64)
9366
0
    return 0;
9367
0
  if ((Subtarget->hasAVX512())) {
9368
0
    return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op1);
9369
0
  }
9370
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9371
0
    return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op1);
9372
0
  }
9373
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9374
0
    return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op1);
9375
0
  }
9376
0
  if ((!Subtarget->hasSSE2())) {
9377
0
    return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op1);
9378
0
  }
9379
0
  return 0;
9380
0
}
9381
9382
0
unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9383
0
  if (RetVT.SimpleTy != MVT::f80)
9384
0
    return 0;
9385
0
  if ((Subtarget->hasX87())) {
9386
0
    return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op1);
9387
0
  }
9388
0
  return 0;
9389
0
}
9390
9391
0
unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9392
0
  if (RetVT.SimpleTy != MVT::v8f16)
9393
0
    return 0;
9394
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9395
0
    return fastEmitInst_rr(X86::VADDPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
9396
0
  }
9397
0
  return 0;
9398
0
}
9399
9400
0
unsigned fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9401
0
  if (RetVT.SimpleTy != MVT::v16f16)
9402
0
    return 0;
9403
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9404
0
    return fastEmitInst_rr(X86::VADDPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
9405
0
  }
9406
0
  return 0;
9407
0
}
9408
9409
0
unsigned fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9410
0
  if (RetVT.SimpleTy != MVT::v32f16)
9411
0
    return 0;
9412
0
  if ((Subtarget->hasFP16())) {
9413
0
    return fastEmitInst_rr(X86::VADDPHZrr, &X86::VR512RegClass, Op0, Op1);
9414
0
  }
9415
0
  return 0;
9416
0
}
9417
9418
0
unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9419
0
  if (RetVT.SimpleTy != MVT::v4f32)
9420
0
    return 0;
9421
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9422
0
    return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
9423
0
  }
9424
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9425
0
    return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op1);
9426
0
  }
9427
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9428
0
    return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op1);
9429
0
  }
9430
0
  return 0;
9431
0
}
9432
9433
0
unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9434
0
  if (RetVT.SimpleTy != MVT::v8f32)
9435
0
    return 0;
9436
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9437
0
    return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
9438
0
  }
9439
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9440
0
    return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op1);
9441
0
  }
9442
0
  return 0;
9443
0
}
9444
9445
0
unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9446
0
  if (RetVT.SimpleTy != MVT::v16f32)
9447
0
    return 0;
9448
0
  if ((Subtarget->hasAVX512())) {
9449
0
    return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op1);
9450
0
  }
9451
0
  return 0;
9452
0
}
9453
9454
0
unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9455
0
  if (RetVT.SimpleTy != MVT::v2f64)
9456
0
    return 0;
9457
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9458
0
    return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
9459
0
  }
9460
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9461
0
    return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op1);
9462
0
  }
9463
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9464
0
    return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op1);
9465
0
  }
9466
0
  return 0;
9467
0
}
9468
9469
0
unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9470
0
  if (RetVT.SimpleTy != MVT::v4f64)
9471
0
    return 0;
9472
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9473
0
    return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
9474
0
  }
9475
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9476
0
    return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op1);
9477
0
  }
9478
0
  return 0;
9479
0
}
9480
9481
0
unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9482
0
  if (RetVT.SimpleTy != MVT::v8f64)
9483
0
    return 0;
9484
0
  if ((Subtarget->hasAVX512())) {
9485
0
    return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op1);
9486
0
  }
9487
0
  return 0;
9488
0
}
9489
9490
0
unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9491
0
  switch (VT.SimpleTy) {
9492
0
  case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
9493
0
  case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
9494
0
  case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
9495
0
  case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1);
9496
0
  case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
9497
0
  case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
9498
0
  case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
9499
0
  case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
9500
0
  case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
9501
0
  case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
9502
0
  case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
9503
0
  case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
9504
0
  case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
9505
0
  default: return 0;
9506
0
  }
9507
0
}
9508
9509
// FastEmit functions for ISD::FDIV.
9510
9511
0
unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9512
0
  if (RetVT.SimpleTy != MVT::f16)
9513
0
    return 0;
9514
0
  if ((Subtarget->hasFP16())) {
9515
0
    return fastEmitInst_rr(X86::VDIVSHZrr, &X86::FR16XRegClass, Op0, Op1);
9516
0
  }
9517
0
  return 0;
9518
0
}
9519
9520
0
unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9521
0
  if (RetVT.SimpleTy != MVT::f32)
9522
0
    return 0;
9523
0
  if ((Subtarget->hasAVX512())) {
9524
0
    return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op1);
9525
0
  }
9526
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9527
0
    return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op1);
9528
0
  }
9529
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9530
0
    return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op1);
9531
0
  }
9532
0
  if ((!Subtarget->hasSSE1())) {
9533
0
    return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op1);
9534
0
  }
9535
0
  return 0;
9536
0
}
9537
9538
0
unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9539
0
  if (RetVT.SimpleTy != MVT::f64)
9540
0
    return 0;
9541
0
  if ((Subtarget->hasAVX512())) {
9542
0
    return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op1);
9543
0
  }
9544
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9545
0
    return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op1);
9546
0
  }
9547
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9548
0
    return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op1);
9549
0
  }
9550
0
  if ((!Subtarget->hasSSE2())) {
9551
0
    return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op1);
9552
0
  }
9553
0
  return 0;
9554
0
}
9555
9556
0
unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9557
0
  if (RetVT.SimpleTy != MVT::f80)
9558
0
    return 0;
9559
0
  if ((Subtarget->hasX87())) {
9560
0
    return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op1);
9561
0
  }
9562
0
  return 0;
9563
0
}
9564
9565
0
unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9566
0
  if (RetVT.SimpleTy != MVT::v8f16)
9567
0
    return 0;
9568
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9569
0
    return fastEmitInst_rr(X86::VDIVPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
9570
0
  }
9571
0
  return 0;
9572
0
}
9573
9574
0
unsigned fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9575
0
  if (RetVT.SimpleTy != MVT::v16f16)
9576
0
    return 0;
9577
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9578
0
    return fastEmitInst_rr(X86::VDIVPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
9579
0
  }
9580
0
  return 0;
9581
0
}
9582
9583
0
unsigned fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9584
0
  if (RetVT.SimpleTy != MVT::v32f16)
9585
0
    return 0;
9586
0
  if ((Subtarget->hasFP16())) {
9587
0
    return fastEmitInst_rr(X86::VDIVPHZrr, &X86::VR512RegClass, Op0, Op1);
9588
0
  }
9589
0
  return 0;
9590
0
}
9591
9592
0
unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9593
0
  if (RetVT.SimpleTy != MVT::v4f32)
9594
0
    return 0;
9595
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9596
0
    return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
9597
0
  }
9598
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9599
0
    return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op1);
9600
0
  }
9601
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9602
0
    return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op1);
9603
0
  }
9604
0
  return 0;
9605
0
}
9606
9607
0
unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9608
0
  if (RetVT.SimpleTy != MVT::v8f32)
9609
0
    return 0;
9610
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9611
0
    return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
9612
0
  }
9613
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9614
0
    return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op1);
9615
0
  }
9616
0
  return 0;
9617
0
}
9618
9619
0
unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9620
0
  if (RetVT.SimpleTy != MVT::v16f32)
9621
0
    return 0;
9622
0
  if ((Subtarget->hasAVX512())) {
9623
0
    return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op1);
9624
0
  }
9625
0
  return 0;
9626
0
}
9627
9628
0
unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9629
0
  if (RetVT.SimpleTy != MVT::v2f64)
9630
0
    return 0;
9631
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9632
0
    return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
9633
0
  }
9634
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9635
0
    return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op1);
9636
0
  }
9637
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9638
0
    return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op1);
9639
0
  }
9640
0
  return 0;
9641
0
}
9642
9643
0
unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9644
0
  if (RetVT.SimpleTy != MVT::v4f64)
9645
0
    return 0;
9646
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9647
0
    return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
9648
0
  }
9649
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9650
0
    return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op1);
9651
0
  }
9652
0
  return 0;
9653
0
}
9654
9655
0
unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9656
0
  if (RetVT.SimpleTy != MVT::v8f64)
9657
0
    return 0;
9658
0
  if ((Subtarget->hasAVX512())) {
9659
0
    return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op1);
9660
0
  }
9661
0
  return 0;
9662
0
}
9663
9664
0
unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9665
0
  switch (VT.SimpleTy) {
9666
0
  case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
9667
0
  case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
9668
0
  case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
9669
0
  case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
9670
0
  case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
9671
0
  case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
9672
0
  case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
9673
0
  case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
9674
0
  case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
9675
0
  case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
9676
0
  case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
9677
0
  case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
9678
0
  case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
9679
0
  default: return 0;
9680
0
  }
9681
0
}
9682
9683
// FastEmit functions for ISD::FMUL.
9684
9685
0
unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9686
0
  if (RetVT.SimpleTy != MVT::f16)
9687
0
    return 0;
9688
0
  if ((Subtarget->hasFP16())) {
9689
0
    return fastEmitInst_rr(X86::VMULSHZrr, &X86::FR16XRegClass, Op0, Op1);
9690
0
  }
9691
0
  return 0;
9692
0
}
9693
9694
0
unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9695
0
  if (RetVT.SimpleTy != MVT::f32)
9696
0
    return 0;
9697
0
  if ((Subtarget->hasAVX512())) {
9698
0
    return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op1);
9699
0
  }
9700
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9701
0
    return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op1);
9702
0
  }
9703
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9704
0
    return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op1);
9705
0
  }
9706
0
  if ((!Subtarget->hasSSE1())) {
9707
0
    return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op1);
9708
0
  }
9709
0
  return 0;
9710
0
}
9711
9712
0
unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9713
0
  if (RetVT.SimpleTy != MVT::f64)
9714
0
    return 0;
9715
0
  if ((Subtarget->hasAVX512())) {
9716
0
    return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op1);
9717
0
  }
9718
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9719
0
    return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op1);
9720
0
  }
9721
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9722
0
    return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op1);
9723
0
  }
9724
0
  if ((!Subtarget->hasSSE2())) {
9725
0
    return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op1);
9726
0
  }
9727
0
  return 0;
9728
0
}
9729
9730
0
unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9731
0
  if (RetVT.SimpleTy != MVT::f80)
9732
0
    return 0;
9733
0
  if ((Subtarget->hasX87())) {
9734
0
    return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op1);
9735
0
  }
9736
0
  return 0;
9737
0
}
9738
9739
0
unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9740
0
  if (RetVT.SimpleTy != MVT::v8f16)
9741
0
    return 0;
9742
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9743
0
    return fastEmitInst_rr(X86::VMULPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
9744
0
  }
9745
0
  return 0;
9746
0
}
9747
9748
0
unsigned fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9749
0
  if (RetVT.SimpleTy != MVT::v16f16)
9750
0
    return 0;
9751
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9752
0
    return fastEmitInst_rr(X86::VMULPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
9753
0
  }
9754
0
  return 0;
9755
0
}
9756
9757
0
unsigned fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9758
0
  if (RetVT.SimpleTy != MVT::v32f16)
9759
0
    return 0;
9760
0
  if ((Subtarget->hasFP16())) {
9761
0
    return fastEmitInst_rr(X86::VMULPHZrr, &X86::VR512RegClass, Op0, Op1);
9762
0
  }
9763
0
  return 0;
9764
0
}
9765
9766
0
unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9767
0
  if (RetVT.SimpleTy != MVT::v4f32)
9768
0
    return 0;
9769
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9770
0
    return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
9771
0
  }
9772
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9773
0
    return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op1);
9774
0
  }
9775
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9776
0
    return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op1);
9777
0
  }
9778
0
  return 0;
9779
0
}
9780
9781
0
unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9782
0
  if (RetVT.SimpleTy != MVT::v8f32)
9783
0
    return 0;
9784
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9785
0
    return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
9786
0
  }
9787
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9788
0
    return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op1);
9789
0
  }
9790
0
  return 0;
9791
0
}
9792
9793
0
unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9794
0
  if (RetVT.SimpleTy != MVT::v16f32)
9795
0
    return 0;
9796
0
  if ((Subtarget->hasAVX512())) {
9797
0
    return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op1);
9798
0
  }
9799
0
  return 0;
9800
0
}
9801
9802
0
unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9803
0
  if (RetVT.SimpleTy != MVT::v2f64)
9804
0
    return 0;
9805
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9806
0
    return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
9807
0
  }
9808
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9809
0
    return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op1);
9810
0
  }
9811
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9812
0
    return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op1);
9813
0
  }
9814
0
  return 0;
9815
0
}
9816
9817
0
unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9818
0
  if (RetVT.SimpleTy != MVT::v4f64)
9819
0
    return 0;
9820
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9821
0
    return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
9822
0
  }
9823
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9824
0
    return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op1);
9825
0
  }
9826
0
  return 0;
9827
0
}
9828
9829
0
unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9830
0
  if (RetVT.SimpleTy != MVT::v8f64)
9831
0
    return 0;
9832
0
  if ((Subtarget->hasAVX512())) {
9833
0
    return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op1);
9834
0
  }
9835
0
  return 0;
9836
0
}
9837
9838
0
unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9839
0
  switch (VT.SimpleTy) {
9840
0
  case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
9841
0
  case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
9842
0
  case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
9843
0
  case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
9844
0
  case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
9845
0
  case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
9846
0
  case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
9847
0
  case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
9848
0
  case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
9849
0
  case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
9850
0
  case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
9851
0
  case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
9852
0
  case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
9853
0
  default: return 0;
9854
0
  }
9855
0
}
9856
9857
// FastEmit functions for ISD::FSUB.
9858
9859
0
unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9860
0
  if (RetVT.SimpleTy != MVT::f16)
9861
0
    return 0;
9862
0
  if ((Subtarget->hasFP16())) {
9863
0
    return fastEmitInst_rr(X86::VSUBSHZrr, &X86::FR16XRegClass, Op0, Op1);
9864
0
  }
9865
0
  return 0;
9866
0
}
9867
9868
0
unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9869
0
  if (RetVT.SimpleTy != MVT::f32)
9870
0
    return 0;
9871
0
  if ((Subtarget->hasAVX512())) {
9872
0
    return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op1);
9873
0
  }
9874
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9875
0
    return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op1);
9876
0
  }
9877
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9878
0
    return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op1);
9879
0
  }
9880
0
  if ((!Subtarget->hasSSE1())) {
9881
0
    return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op1);
9882
0
  }
9883
0
  return 0;
9884
0
}
9885
9886
0
unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9887
0
  if (RetVT.SimpleTy != MVT::f64)
9888
0
    return 0;
9889
0
  if ((Subtarget->hasAVX512())) {
9890
0
    return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op1);
9891
0
  }
9892
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9893
0
    return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op1);
9894
0
  }
9895
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9896
0
    return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op1);
9897
0
  }
9898
0
  if ((!Subtarget->hasSSE2())) {
9899
0
    return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op1);
9900
0
  }
9901
0
  return 0;
9902
0
}
9903
9904
0
unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9905
0
  if (RetVT.SimpleTy != MVT::f80)
9906
0
    return 0;
9907
0
  if ((Subtarget->hasX87())) {
9908
0
    return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op1);
9909
0
  }
9910
0
  return 0;
9911
0
}
9912
9913
0
unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9914
0
  if (RetVT.SimpleTy != MVT::v8f16)
9915
0
    return 0;
9916
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9917
0
    return fastEmitInst_rr(X86::VSUBPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
9918
0
  }
9919
0
  return 0;
9920
0
}
9921
9922
0
unsigned fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9923
0
  if (RetVT.SimpleTy != MVT::v16f16)
9924
0
    return 0;
9925
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
9926
0
    return fastEmitInst_rr(X86::VSUBPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
9927
0
  }
9928
0
  return 0;
9929
0
}
9930
9931
0
unsigned fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9932
0
  if (RetVT.SimpleTy != MVT::v32f16)
9933
0
    return 0;
9934
0
  if ((Subtarget->hasFP16())) {
9935
0
    return fastEmitInst_rr(X86::VSUBPHZrr, &X86::VR512RegClass, Op0, Op1);
9936
0
  }
9937
0
  return 0;
9938
0
}
9939
9940
0
unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9941
0
  if (RetVT.SimpleTy != MVT::v4f32)
9942
0
    return 0;
9943
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9944
0
    return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
9945
0
  }
9946
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9947
0
    return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op1);
9948
0
  }
9949
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9950
0
    return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op1);
9951
0
  }
9952
0
  return 0;
9953
0
}
9954
9955
0
unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9956
0
  if (RetVT.SimpleTy != MVT::v8f32)
9957
0
    return 0;
9958
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9959
0
    return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
9960
0
  }
9961
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9962
0
    return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op1);
9963
0
  }
9964
0
  return 0;
9965
0
}
9966
9967
0
unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9968
0
  if (RetVT.SimpleTy != MVT::v16f32)
9969
0
    return 0;
9970
0
  if ((Subtarget->hasAVX512())) {
9971
0
    return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op1);
9972
0
  }
9973
0
  return 0;
9974
0
}
9975
9976
0
unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9977
0
  if (RetVT.SimpleTy != MVT::v2f64)
9978
0
    return 0;
9979
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9980
0
    return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
9981
0
  }
9982
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9983
0
    return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op1);
9984
0
  }
9985
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9986
0
    return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op1);
9987
0
  }
9988
0
  return 0;
9989
0
}
9990
9991
0
unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9992
0
  if (RetVT.SimpleTy != MVT::v4f64)
9993
0
    return 0;
9994
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9995
0
    return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
9996
0
  }
9997
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9998
0
    return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op1);
9999
0
  }
10000
0
  return 0;
10001
0
}
10002
10003
0
unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10004
0
  if (RetVT.SimpleTy != MVT::v8f64)
10005
0
    return 0;
10006
0
  if ((Subtarget->hasAVX512())) {
10007
0
    return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op1);
10008
0
  }
10009
0
  return 0;
10010
0
}
10011
10012
0
unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10013
0
  switch (VT.SimpleTy) {
10014
0
  case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
10015
0
  case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
10016
0
  case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
10017
0
  case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
10018
0
  case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
10019
0
  case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
10020
0
  case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
10021
0
  case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
10022
0
  case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
10023
0
  case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
10024
0
  case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
10025
0
  case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
10026
0
  case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
10027
0
  default: return 0;
10028
0
  }
10029
0
}
10030
10031
// FastEmit functions for ISD::MUL.
10032
10033
0
unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10034
0
  if (RetVT.SimpleTy != MVT::i8)
10035
0
    return 0;
10036
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
10037
0
  return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1);
10038
0
}
10039
10040
0
unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10041
0
  if (RetVT.SimpleTy != MVT::i16)
10042
0
    return 0;
10043
0
  if ((Subtarget->hasNDD())) {
10044
0
    return fastEmitInst_rr(X86::IMUL16rr_ND, &X86::GR16RegClass, Op0, Op1);
10045
0
  }
10046
0
  if ((!Subtarget->hasNDD())) {
10047
0
    return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op1);
10048
0
  }
10049
0
  return 0;
10050
0
}
10051
10052
0
unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10053
0
  if (RetVT.SimpleTy != MVT::i32)
10054
0
    return 0;
10055
0
  if ((Subtarget->hasNDD())) {
10056
0
    return fastEmitInst_rr(X86::IMUL32rr_ND, &X86::GR32RegClass, Op0, Op1);
10057
0
  }
10058
0
  if ((!Subtarget->hasNDD())) {
10059
0
    return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op1);
10060
0
  }
10061
0
  return 0;
10062
0
}
10063
10064
0
unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10065
0
  if (RetVT.SimpleTy != MVT::i64)
10066
0
    return 0;
10067
0
  if ((Subtarget->hasNDD())) {
10068
0
    return fastEmitInst_rr(X86::IMUL64rr_ND, &X86::GR64RegClass, Op0, Op1);
10069
0
  }
10070
0
  if ((!Subtarget->hasNDD())) {
10071
0
    return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op1);
10072
0
  }
10073
0
  return 0;
10074
0
}
10075
10076
0
unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10077
0
  if (RetVT.SimpleTy != MVT::v8i16)
10078
0
    return 0;
10079
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10080
0
    return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op1);
10081
0
  }
10082
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10083
0
    return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op1);
10084
0
  }
10085
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10086
0
    return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op1);
10087
0
  }
10088
0
  return 0;
10089
0
}
10090
10091
0
unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10092
0
  if (RetVT.SimpleTy != MVT::v16i16)
10093
0
    return 0;
10094
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10095
0
    return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op1);
10096
0
  }
10097
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10098
0
    return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op1);
10099
0
  }
10100
0
  return 0;
10101
0
}
10102
10103
0
unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10104
0
  if (RetVT.SimpleTy != MVT::v32i16)
10105
0
    return 0;
10106
0
  if ((Subtarget->hasBWI())) {
10107
0
    return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op1);
10108
0
  }
10109
0
  return 0;
10110
0
}
10111
10112
0
unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10113
0
  if (RetVT.SimpleTy != MVT::v4i32)
10114
0
    return 0;
10115
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10116
0
    return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op1);
10117
0
  }
10118
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10119
0
    return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op1);
10120
0
  }
10121
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10122
0
    return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op1);
10123
0
  }
10124
0
  return 0;
10125
0
}
10126
10127
0
unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10128
0
  if (RetVT.SimpleTy != MVT::v8i32)
10129
0
    return 0;
10130
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10131
0
    return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op1);
10132
0
  }
10133
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10134
0
    return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op1);
10135
0
  }
10136
0
  return 0;
10137
0
}
10138
10139
0
unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10140
0
  if (RetVT.SimpleTy != MVT::v16i32)
10141
0
    return 0;
10142
0
  if ((Subtarget->hasAVX512())) {
10143
0
    return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op1);
10144
0
  }
10145
0
  return 0;
10146
0
}
10147
10148
0
unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10149
0
  if (RetVT.SimpleTy != MVT::v2i64)
10150
0
    return 0;
10151
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
10152
0
    return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10153
0
  }
10154
0
  return 0;
10155
0
}
10156
10157
0
unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10158
0
  if (RetVT.SimpleTy != MVT::v4i64)
10159
0
    return 0;
10160
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
10161
0
    return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10162
0
  }
10163
0
  return 0;
10164
0
}
10165
10166
0
unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10167
0
  if (RetVT.SimpleTy != MVT::v8i64)
10168
0
    return 0;
10169
0
  if ((Subtarget->hasDQI())) {
10170
0
    return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op1);
10171
0
  }
10172
0
  return 0;
10173
0
}
10174
10175
0
unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10176
0
  switch (VT.SimpleTy) {
10177
0
  case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1);
10178
0
  case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1);
10179
0
  case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
10180
0
  case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1);
10181
0
  case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
10182
0
  case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1);
10183
0
  case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1);
10184
0
  case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
10185
0
  case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1);
10186
0
  case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1);
10187
0
  case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1);
10188
0
  case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1);
10189
0
  case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1);
10190
0
  default: return 0;
10191
0
  }
10192
0
}
10193
10194
// FastEmit functions for ISD::MULHS.
10195
10196
0
unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10197
0
  if (RetVT.SimpleTy != MVT::v8i16)
10198
0
    return 0;
10199
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10200
0
    return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op1);
10201
0
  }
10202
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10203
0
    return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op1);
10204
0
  }
10205
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10206
0
    return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op1);
10207
0
  }
10208
0
  return 0;
10209
0
}
10210
10211
0
unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10212
0
  if (RetVT.SimpleTy != MVT::v16i16)
10213
0
    return 0;
10214
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10215
0
    return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op1);
10216
0
  }
10217
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10218
0
    return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op1);
10219
0
  }
10220
0
  return 0;
10221
0
}
10222
10223
0
unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10224
0
  if (RetVT.SimpleTy != MVT::v32i16)
10225
0
    return 0;
10226
0
  if ((Subtarget->hasBWI())) {
10227
0
    return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op1);
10228
0
  }
10229
0
  return 0;
10230
0
}
10231
10232
0
unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10233
0
  switch (VT.SimpleTy) {
10234
0
  case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
10235
0
  case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1);
10236
0
  case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1);
10237
0
  default: return 0;
10238
0
  }
10239
0
}
10240
10241
// FastEmit functions for ISD::MULHU.
10242
10243
0
unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10244
0
  if (RetVT.SimpleTy != MVT::v8i16)
10245
0
    return 0;
10246
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10247
0
    return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op1);
10248
0
  }
10249
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10250
0
    return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op1);
10251
0
  }
10252
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10253
0
    return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op1);
10254
0
  }
10255
0
  return 0;
10256
0
}
10257
10258
0
unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10259
0
  if (RetVT.SimpleTy != MVT::v16i16)
10260
0
    return 0;
10261
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10262
0
    return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op1);
10263
0
  }
10264
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10265
0
    return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op1);
10266
0
  }
10267
0
  return 0;
10268
0
}
10269
10270
0
unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10271
0
  if (RetVT.SimpleTy != MVT::v32i16)
10272
0
    return 0;
10273
0
  if ((Subtarget->hasBWI())) {
10274
0
    return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op1);
10275
0
  }
10276
0
  return 0;
10277
0
}
10278
10279
0
unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10280
0
  switch (VT.SimpleTy) {
10281
0
  case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
10282
0
  case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1);
10283
0
  case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1);
10284
0
  default: return 0;
10285
0
  }
10286
0
}
10287
10288
// FastEmit functions for ISD::OR.
10289
10290
0
unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10291
0
  if (RetVT.SimpleTy != MVT::i8)
10292
0
    return 0;
10293
0
  if ((Subtarget->hasNDD())) {
10294
0
    return fastEmitInst_rr(X86::OR8rr_ND, &X86::GR8RegClass, Op0, Op1);
10295
0
  }
10296
0
  if ((!Subtarget->hasNDD())) {
10297
0
    return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op1);
10298
0
  }
10299
0
  return 0;
10300
0
}
10301
10302
0
unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10303
0
  if (RetVT.SimpleTy != MVT::i16)
10304
0
    return 0;
10305
0
  if ((Subtarget->hasNDD())) {
10306
0
    return fastEmitInst_rr(X86::OR16rr_ND, &X86::GR16RegClass, Op0, Op1);
10307
0
  }
10308
0
  if ((!Subtarget->hasNDD())) {
10309
0
    return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op1);
10310
0
  }
10311
0
  return 0;
10312
0
}
10313
10314
0
unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10315
0
  if (RetVT.SimpleTy != MVT::i32)
10316
0
    return 0;
10317
0
  if ((Subtarget->hasNDD())) {
10318
0
    return fastEmitInst_rr(X86::OR32rr_ND, &X86::GR32RegClass, Op0, Op1);
10319
0
  }
10320
0
  if ((!Subtarget->hasNDD())) {
10321
0
    return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op1);
10322
0
  }
10323
0
  return 0;
10324
0
}
10325
10326
0
unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10327
0
  if (RetVT.SimpleTy != MVT::i64)
10328
0
    return 0;
10329
0
  if ((Subtarget->hasNDD())) {
10330
0
    return fastEmitInst_rr(X86::OR64rr_ND, &X86::GR64RegClass, Op0, Op1);
10331
0
  }
10332
0
  if ((!Subtarget->hasNDD())) {
10333
0
    return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op1);
10334
0
  }
10335
0
  return 0;
10336
0
}
10337
10338
0
unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10339
0
  if (RetVT.SimpleTy != MVT::v8i1)
10340
0
    return 0;
10341
0
  if ((Subtarget->hasDQI())) {
10342
0
    return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op1);
10343
0
  }
10344
0
  return 0;
10345
0
}
10346
10347
0
unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10348
0
  if (RetVT.SimpleTy != MVT::v16i1)
10349
0
    return 0;
10350
0
  if ((Subtarget->hasAVX512())) {
10351
0
    return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op1);
10352
0
  }
10353
0
  return 0;
10354
0
}
10355
10356
0
unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10357
0
  if (RetVT.SimpleTy != MVT::v32i1)
10358
0
    return 0;
10359
0
  if ((Subtarget->hasBWI())) {
10360
0
    return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op1);
10361
0
  }
10362
0
  return 0;
10363
0
}
10364
10365
0
unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10366
0
  if (RetVT.SimpleTy != MVT::v64i1)
10367
0
    return 0;
10368
0
  if ((Subtarget->hasBWI())) {
10369
0
    return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op1);
10370
0
  }
10371
0
  return 0;
10372
0
}
10373
10374
0
unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10375
0
  if (RetVT.SimpleTy != MVT::v16i8)
10376
0
    return 0;
10377
0
  if ((Subtarget->hasVLX())) {
10378
0
    return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10379
0
  }
10380
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10381
0
    return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1);
10382
0
  }
10383
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10384
0
    return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1);
10385
0
  }
10386
0
  return 0;
10387
0
}
10388
10389
0
unsigned fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10390
0
  if (RetVT.SimpleTy != MVT::v32i8)
10391
0
    return 0;
10392
0
  if ((Subtarget->hasVLX())) {
10393
0
    return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10394
0
  }
10395
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10396
0
    return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1);
10397
0
  }
10398
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10399
0
    return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1);
10400
0
  }
10401
0
  return 0;
10402
0
}
10403
10404
0
unsigned fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10405
0
  if (RetVT.SimpleTy != MVT::v64i8)
10406
0
    return 0;
10407
0
  if ((Subtarget->hasAVX512())) {
10408
0
    return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1);
10409
0
  }
10410
0
  return 0;
10411
0
}
10412
10413
0
unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10414
0
  if (RetVT.SimpleTy != MVT::v8i16)
10415
0
    return 0;
10416
0
  if ((Subtarget->hasVLX())) {
10417
0
    return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10418
0
  }
10419
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10420
0
    return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1);
10421
0
  }
10422
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10423
0
    return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1);
10424
0
  }
10425
0
  return 0;
10426
0
}
10427
10428
0
unsigned fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10429
0
  if (RetVT.SimpleTy != MVT::v16i16)
10430
0
    return 0;
10431
0
  if ((Subtarget->hasVLX())) {
10432
0
    return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10433
0
  }
10434
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10435
0
    return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1);
10436
0
  }
10437
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10438
0
    return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1);
10439
0
  }
10440
0
  return 0;
10441
0
}
10442
10443
0
unsigned fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10444
0
  if (RetVT.SimpleTy != MVT::v32i16)
10445
0
    return 0;
10446
0
  if ((Subtarget->hasAVX512())) {
10447
0
    return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1);
10448
0
  }
10449
0
  return 0;
10450
0
}
10451
10452
0
unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10453
0
  if (RetVT.SimpleTy != MVT::v4i32)
10454
0
    return 0;
10455
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10456
0
    return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1);
10457
0
  }
10458
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10459
0
    return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1);
10460
0
  }
10461
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10462
0
    return fastEmitInst_rr(X86::VPORDZ128rr, &X86::VR128XRegClass, Op0, Op1);
10463
0
  }
10464
0
  return 0;
10465
0
}
10466
10467
0
unsigned fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10468
0
  if (RetVT.SimpleTy != MVT::v8i32)
10469
0
    return 0;
10470
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10471
0
    return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1);
10472
0
  }
10473
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10474
0
    return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1);
10475
0
  }
10476
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10477
0
    return fastEmitInst_rr(X86::VPORDZ256rr, &X86::VR256XRegClass, Op0, Op1);
10478
0
  }
10479
0
  return 0;
10480
0
}
10481
10482
0
unsigned fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10483
0
  if (RetVT.SimpleTy != MVT::v16i32)
10484
0
    return 0;
10485
0
  if ((Subtarget->hasAVX512())) {
10486
0
    return fastEmitInst_rr(X86::VPORDZrr, &X86::VR512RegClass, Op0, Op1);
10487
0
  }
10488
0
  return 0;
10489
0
}
10490
10491
0
unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10492
0
  if (RetVT.SimpleTy != MVT::v2i64)
10493
0
    return 0;
10494
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10495
0
    return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10496
0
  }
10497
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10498
0
    return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1);
10499
0
  }
10500
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10501
0
    return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1);
10502
0
  }
10503
0
  return 0;
10504
0
}
10505
10506
0
unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10507
0
  if (RetVT.SimpleTy != MVT::v4i64)
10508
0
    return 0;
10509
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10510
0
    return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1);
10511
0
  }
10512
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10513
0
    return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10514
0
  }
10515
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10516
0
    return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1);
10517
0
  }
10518
0
  return 0;
10519
0
}
10520
10521
0
unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10522
0
  if (RetVT.SimpleTy != MVT::v8i64)
10523
0
    return 0;
10524
0
  if ((Subtarget->hasAVX512())) {
10525
0
    return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1);
10526
0
  }
10527
0
  return 0;
10528
0
}
10529
10530
0
unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10531
0
  switch (VT.SimpleTy) {
10532
0
  case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1);
10533
0
  case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1);
10534
0
  case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
10535
0
  case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
10536
0
  case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1);
10537
0
  case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1);
10538
0
  case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1);
10539
0
  case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1);
10540
0
  case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
10541
0
  case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1);
10542
0
  case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1);
10543
0
  case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
10544
0
  case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1);
10545
0
  case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1);
10546
0
  case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
10547
0
  case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1);
10548
0
  case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1);
10549
0
  case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
10550
0
  case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1);
10551
0
  case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1);
10552
0
  default: return 0;
10553
0
  }
10554
0
}
10555
10556
// FastEmit functions for ISD::ROTL.
10557
10558
0
unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10559
0
  if (RetVT.SimpleTy != MVT::i8)
10560
0
    return 0;
10561
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
10562
0
  return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0);
10563
0
}
10564
10565
0
unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10566
0
  if (RetVT.SimpleTy != MVT::v16i8)
10567
0
    return 0;
10568
0
  if ((Subtarget->hasXOP())) {
10569
0
    return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op1);
10570
0
  }
10571
0
  return 0;
10572
0
}
10573
10574
0
unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10575
0
  if (RetVT.SimpleTy != MVT::v8i16)
10576
0
    return 0;
10577
0
  if ((Subtarget->hasXOP())) {
10578
0
    return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op1);
10579
0
  }
10580
0
  return 0;
10581
0
}
10582
10583
0
unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10584
0
  if (RetVT.SimpleTy != MVT::v4i32)
10585
0
    return 0;
10586
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10587
0
    return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op1);
10588
0
  }
10589
0
  if ((Subtarget->hasXOP())) {
10590
0
    return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op1);
10591
0
  }
10592
0
  return 0;
10593
0
}
10594
10595
0
unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10596
0
  if (RetVT.SimpleTy != MVT::v8i32)
10597
0
    return 0;
10598
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10599
0
    return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op1);
10600
0
  }
10601
0
  return 0;
10602
0
}
10603
10604
0
unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10605
0
  if (RetVT.SimpleTy != MVT::v16i32)
10606
0
    return 0;
10607
0
  if ((Subtarget->hasAVX512())) {
10608
0
    return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op1);
10609
0
  }
10610
0
  return 0;
10611
0
}
10612
10613
0
unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10614
0
  if (RetVT.SimpleTy != MVT::v2i64)
10615
0
    return 0;
10616
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10617
0
    return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10618
0
  }
10619
0
  if ((Subtarget->hasXOP())) {
10620
0
    return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op1);
10621
0
  }
10622
0
  return 0;
10623
0
}
10624
10625
0
unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10626
0
  if (RetVT.SimpleTy != MVT::v4i64)
10627
0
    return 0;
10628
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10629
0
    return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10630
0
  }
10631
0
  return 0;
10632
0
}
10633
10634
0
unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10635
0
  if (RetVT.SimpleTy != MVT::v8i64)
10636
0
    return 0;
10637
0
  if ((Subtarget->hasAVX512())) {
10638
0
    return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op1);
10639
0
  }
10640
0
  return 0;
10641
0
}
10642
10643
0
unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10644
0
  switch (VT.SimpleTy) {
10645
0
  case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1);
10646
0
  case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1);
10647
0
  case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1);
10648
0
  case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1);
10649
0
  case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1);
10650
0
  case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1);
10651
0
  case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1);
10652
0
  case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1);
10653
0
  case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1);
10654
0
  default: return 0;
10655
0
  }
10656
0
}
10657
10658
// FastEmit functions for ISD::ROTR.
10659
10660
0
unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10661
0
  if (RetVT.SimpleTy != MVT::i8)
10662
0
    return 0;
10663
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
10664
0
  return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0);
10665
0
}
10666
10667
0
unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10668
0
  if (RetVT.SimpleTy != MVT::v4i32)
10669
0
    return 0;
10670
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10671
0
    return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op1);
10672
0
  }
10673
0
  return 0;
10674
0
}
10675
10676
0
unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10677
0
  if (RetVT.SimpleTy != MVT::v8i32)
10678
0
    return 0;
10679
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10680
0
    return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op1);
10681
0
  }
10682
0
  return 0;
10683
0
}
10684
10685
0
unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10686
0
  if (RetVT.SimpleTy != MVT::v16i32)
10687
0
    return 0;
10688
0
  if ((Subtarget->hasAVX512())) {
10689
0
    return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op1);
10690
0
  }
10691
0
  return 0;
10692
0
}
10693
10694
0
unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10695
0
  if (RetVT.SimpleTy != MVT::v2i64)
10696
0
    return 0;
10697
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10698
0
    return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10699
0
  }
10700
0
  return 0;
10701
0
}
10702
10703
0
unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10704
0
  if (RetVT.SimpleTy != MVT::v4i64)
10705
0
    return 0;
10706
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10707
0
    return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10708
0
  }
10709
0
  return 0;
10710
0
}
10711
10712
0
unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10713
0
  if (RetVT.SimpleTy != MVT::v8i64)
10714
0
    return 0;
10715
0
  if ((Subtarget->hasAVX512())) {
10716
0
    return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op1);
10717
0
  }
10718
0
  return 0;
10719
0
}
10720
10721
0
unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10722
0
  switch (VT.SimpleTy) {
10723
0
  case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1);
10724
0
  case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1);
10725
0
  case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1);
10726
0
  case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1);
10727
0
  case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1);
10728
0
  case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1);
10729
0
  case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1);
10730
0
  default: return 0;
10731
0
  }
10732
0
}
10733
10734
// FastEmit functions for ISD::SADDSAT.
10735
10736
0
unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10737
0
  if (RetVT.SimpleTy != MVT::v16i8)
10738
0
    return 0;
10739
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10740
0
    return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
10741
0
  }
10742
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10743
0
    return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op1);
10744
0
  }
10745
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10746
0
    return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op1);
10747
0
  }
10748
0
  return 0;
10749
0
}
10750
10751
0
unsigned fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10752
0
  if (RetVT.SimpleTy != MVT::v32i8)
10753
0
    return 0;
10754
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10755
0
    return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
10756
0
  }
10757
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10758
0
    return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op1);
10759
0
  }
10760
0
  return 0;
10761
0
}
10762
10763
0
unsigned fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10764
0
  if (RetVT.SimpleTy != MVT::v64i8)
10765
0
    return 0;
10766
0
  if ((Subtarget->hasBWI())) {
10767
0
    return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op1);
10768
0
  }
10769
0
  return 0;
10770
0
}
10771
10772
0
unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10773
0
  if (RetVT.SimpleTy != MVT::v8i16)
10774
0
    return 0;
10775
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10776
0
    return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
10777
0
  }
10778
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10779
0
    return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op1);
10780
0
  }
10781
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10782
0
    return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op1);
10783
0
  }
10784
0
  return 0;
10785
0
}
10786
10787
0
unsigned fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10788
0
  if (RetVT.SimpleTy != MVT::v16i16)
10789
0
    return 0;
10790
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10791
0
    return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
10792
0
  }
10793
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10794
0
    return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op1);
10795
0
  }
10796
0
  return 0;
10797
0
}
10798
10799
0
unsigned fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10800
0
  if (RetVT.SimpleTy != MVT::v32i16)
10801
0
    return 0;
10802
0
  if ((Subtarget->hasBWI())) {
10803
0
    return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op1);
10804
0
  }
10805
0
  return 0;
10806
0
}
10807
10808
0
unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10809
0
  switch (VT.SimpleTy) {
10810
0
  case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
10811
0
  case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
10812
0
  case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
10813
0
  case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
10814
0
  case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
10815
0
  case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
10816
0
  default: return 0;
10817
0
  }
10818
0
}
10819
10820
// FastEmit functions for ISD::SHL.
10821
10822
0
unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10823
0
  if (RetVT.SimpleTy != MVT::i8)
10824
0
    return 0;
10825
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
10826
0
  return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0);
10827
0
}
10828
10829
0
unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10830
0
  switch (VT.SimpleTy) {
10831
0
  case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1);
10832
0
  default: return 0;
10833
0
  }
10834
0
}
10835
10836
// FastEmit functions for ISD::SMAX.
10837
10838
0
unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10839
0
  if (RetVT.SimpleTy != MVT::v16i8)
10840
0
    return 0;
10841
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10842
0
    return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
10843
0
  }
10844
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10845
0
    return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op1);
10846
0
  }
10847
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10848
0
    return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op1);
10849
0
  }
10850
0
  return 0;
10851
0
}
10852
10853
0
unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10854
0
  if (RetVT.SimpleTy != MVT::v32i8)
10855
0
    return 0;
10856
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10857
0
    return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
10858
0
  }
10859
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10860
0
    return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op1);
10861
0
  }
10862
0
  return 0;
10863
0
}
10864
10865
0
unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10866
0
  if (RetVT.SimpleTy != MVT::v64i8)
10867
0
    return 0;
10868
0
  if ((Subtarget->hasBWI())) {
10869
0
    return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op1);
10870
0
  }
10871
0
  return 0;
10872
0
}
10873
10874
0
unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10875
0
  if (RetVT.SimpleTy != MVT::v8i16)
10876
0
    return 0;
10877
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10878
0
    return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
10879
0
  }
10880
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10881
0
    return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op1);
10882
0
  }
10883
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10884
0
    return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op1);
10885
0
  }
10886
0
  return 0;
10887
0
}
10888
10889
0
unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10890
0
  if (RetVT.SimpleTy != MVT::v16i16)
10891
0
    return 0;
10892
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10893
0
    return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
10894
0
  }
10895
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10896
0
    return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op1);
10897
0
  }
10898
0
  return 0;
10899
0
}
10900
10901
0
unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10902
0
  if (RetVT.SimpleTy != MVT::v32i16)
10903
0
    return 0;
10904
0
  if ((Subtarget->hasBWI())) {
10905
0
    return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op1);
10906
0
  }
10907
0
  return 0;
10908
0
}
10909
10910
0
unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10911
0
  if (RetVT.SimpleTy != MVT::v4i32)
10912
0
    return 0;
10913
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10914
0
    return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op1);
10915
0
  }
10916
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10917
0
    return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op1);
10918
0
  }
10919
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10920
0
    return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op1);
10921
0
  }
10922
0
  return 0;
10923
0
}
10924
10925
0
unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10926
0
  if (RetVT.SimpleTy != MVT::v8i32)
10927
0
    return 0;
10928
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10929
0
    return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op1);
10930
0
  }
10931
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10932
0
    return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op1);
10933
0
  }
10934
0
  return 0;
10935
0
}
10936
10937
0
unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10938
0
  if (RetVT.SimpleTy != MVT::v16i32)
10939
0
    return 0;
10940
0
  if ((Subtarget->hasAVX512())) {
10941
0
    return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op1);
10942
0
  }
10943
0
  return 0;
10944
0
}
10945
10946
0
unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10947
0
  if (RetVT.SimpleTy != MVT::v2i64)
10948
0
    return 0;
10949
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10950
0
    return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op1);
10951
0
  }
10952
0
  return 0;
10953
0
}
10954
10955
0
unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10956
0
  if (RetVT.SimpleTy != MVT::v4i64)
10957
0
    return 0;
10958
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10959
0
    return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op1);
10960
0
  }
10961
0
  return 0;
10962
0
}
10963
10964
0
unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10965
0
  if (RetVT.SimpleTy != MVT::v8i64)
10966
0
    return 0;
10967
0
  if ((Subtarget->hasAVX512())) {
10968
0
    return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op1);
10969
0
  }
10970
0
  return 0;
10971
0
}
10972
10973
0
unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10974
0
  switch (VT.SimpleTy) {
10975
0
  case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
10976
0
  case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
10977
0
  case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
10978
0
  case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
10979
0
  case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
10980
0
  case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
10981
0
  case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
10982
0
  case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
10983
0
  case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
10984
0
  case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
10985
0
  case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
10986
0
  case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
10987
0
  default: return 0;
10988
0
  }
10989
0
}
10990
10991
// FastEmit functions for ISD::SMIN.
10992
10993
0
unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10994
0
  if (RetVT.SimpleTy != MVT::v16i8)
10995
0
    return 0;
10996
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10997
0
    return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
10998
0
  }
10999
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11000
0
    return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op1);
11001
0
  }
11002
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11003
0
    return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op1);
11004
0
  }
11005
0
  return 0;
11006
0
}
11007
11008
0
unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11009
0
  if (RetVT.SimpleTy != MVT::v32i8)
11010
0
    return 0;
11011
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11012
0
    return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
11013
0
  }
11014
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11015
0
    return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op1);
11016
0
  }
11017
0
  return 0;
11018
0
}
11019
11020
0
unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11021
0
  if (RetVT.SimpleTy != MVT::v64i8)
11022
0
    return 0;
11023
0
  if ((Subtarget->hasBWI())) {
11024
0
    return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op1);
11025
0
  }
11026
0
  return 0;
11027
0
}
11028
11029
0
unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11030
0
  if (RetVT.SimpleTy != MVT::v8i16)
11031
0
    return 0;
11032
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11033
0
    return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
11034
0
  }
11035
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11036
0
    return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op1);
11037
0
  }
11038
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11039
0
    return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op1);
11040
0
  }
11041
0
  return 0;
11042
0
}
11043
11044
0
unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11045
0
  if (RetVT.SimpleTy != MVT::v16i16)
11046
0
    return 0;
11047
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11048
0
    return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
11049
0
  }
11050
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11051
0
    return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op1);
11052
0
  }
11053
0
  return 0;
11054
0
}
11055
11056
0
unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11057
0
  if (RetVT.SimpleTy != MVT::v32i16)
11058
0
    return 0;
11059
0
  if ((Subtarget->hasBWI())) {
11060
0
    return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op1);
11061
0
  }
11062
0
  return 0;
11063
0
}
11064
11065
0
unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11066
0
  if (RetVT.SimpleTy != MVT::v4i32)
11067
0
    return 0;
11068
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11069
0
    return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op1);
11070
0
  }
11071
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
11072
0
    return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op1);
11073
0
  }
11074
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11075
0
    return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op1);
11076
0
  }
11077
0
  return 0;
11078
0
}
11079
11080
0
unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11081
0
  if (RetVT.SimpleTy != MVT::v8i32)
11082
0
    return 0;
11083
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11084
0
    return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op1);
11085
0
  }
11086
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11087
0
    return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op1);
11088
0
  }
11089
0
  return 0;
11090
0
}
11091
11092
0
unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11093
0
  if (RetVT.SimpleTy != MVT::v16i32)
11094
0
    return 0;
11095
0
  if ((Subtarget->hasAVX512())) {
11096
0
    return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op1);
11097
0
  }
11098
0
  return 0;
11099
0
}
11100
11101
0
unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11102
0
  if (RetVT.SimpleTy != MVT::v2i64)
11103
0
    return 0;
11104
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11105
0
    return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op1);
11106
0
  }
11107
0
  return 0;
11108
0
}
11109
11110
0
unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11111
0
  if (RetVT.SimpleTy != MVT::v4i64)
11112
0
    return 0;
11113
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11114
0
    return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op1);
11115
0
  }
11116
0
  return 0;
11117
0
}
11118
11119
0
unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11120
0
  if (RetVT.SimpleTy != MVT::v8i64)
11121
0
    return 0;
11122
0
  if ((Subtarget->hasAVX512())) {
11123
0
    return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op1);
11124
0
  }
11125
0
  return 0;
11126
0
}
11127
11128
0
unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11129
0
  switch (VT.SimpleTy) {
11130
0
  case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
11131
0
  case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
11132
0
  case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
11133
0
  case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
11134
0
  case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
11135
0
  case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
11136
0
  case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
11137
0
  case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
11138
0
  case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
11139
0
  case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
11140
0
  case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
11141
0
  case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
11142
0
  default: return 0;
11143
0
  }
11144
0
}
11145
11146
// FastEmit functions for ISD::SRA.
11147
11148
0
unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11149
0
  if (RetVT.SimpleTy != MVT::i8)
11150
0
    return 0;
11151
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
11152
0
  return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0);
11153
0
}
11154
11155
0
unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11156
0
  switch (VT.SimpleTy) {
11157
0
  case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1);
11158
0
  default: return 0;
11159
0
  }
11160
0
}
11161
11162
// FastEmit functions for ISD::SRL.
11163
11164
0
unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11165
0
  if (RetVT.SimpleTy != MVT::i8)
11166
0
    return 0;
11167
0
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
11168
0
  return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0);
11169
0
}
11170
11171
0
unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11172
0
  switch (VT.SimpleTy) {
11173
0
  case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1);
11174
0
  default: return 0;
11175
0
  }
11176
0
}
11177
11178
// FastEmit functions for ISD::SSUBSAT.
11179
11180
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11181
0
  if (RetVT.SimpleTy != MVT::v16i8)
11182
0
    return 0;
11183
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11184
0
    return fastEmitInst_rr(X86::VPSUBSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
11185
0
  }
11186
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11187
0
    return fastEmitInst_rr(X86::PSUBSBrr, &X86::VR128RegClass, Op0, Op1);
11188
0
  }
11189
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11190
0
    return fastEmitInst_rr(X86::VPSUBSBrr, &X86::VR128RegClass, Op0, Op1);
11191
0
  }
11192
0
  return 0;
11193
0
}
11194
11195
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11196
0
  if (RetVT.SimpleTy != MVT::v32i8)
11197
0
    return 0;
11198
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11199
0
    return fastEmitInst_rr(X86::VPSUBSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
11200
0
  }
11201
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11202
0
    return fastEmitInst_rr(X86::VPSUBSBYrr, &X86::VR256RegClass, Op0, Op1);
11203
0
  }
11204
0
  return 0;
11205
0
}
11206
11207
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11208
0
  if (RetVT.SimpleTy != MVT::v64i8)
11209
0
    return 0;
11210
0
  if ((Subtarget->hasBWI())) {
11211
0
    return fastEmitInst_rr(X86::VPSUBSBZrr, &X86::VR512RegClass, Op0, Op1);
11212
0
  }
11213
0
  return 0;
11214
0
}
11215
11216
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11217
0
  if (RetVT.SimpleTy != MVT::v8i16)
11218
0
    return 0;
11219
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11220
0
    return fastEmitInst_rr(X86::VPSUBSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
11221
0
  }
11222
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11223
0
    return fastEmitInst_rr(X86::PSUBSWrr, &X86::VR128RegClass, Op0, Op1);
11224
0
  }
11225
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11226
0
    return fastEmitInst_rr(X86::VPSUBSWrr, &X86::VR128RegClass, Op0, Op1);
11227
0
  }
11228
0
  return 0;
11229
0
}
11230
11231
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11232
0
  if (RetVT.SimpleTy != MVT::v16i16)
11233
0
    return 0;
11234
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11235
0
    return fastEmitInst_rr(X86::VPSUBSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
11236
0
  }
11237
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11238
0
    return fastEmitInst_rr(X86::VPSUBSWYrr, &X86::VR256RegClass, Op0, Op1);
11239
0
  }
11240
0
  return 0;
11241
0
}
11242
11243
0
unsigned fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11244
0
  if (RetVT.SimpleTy != MVT::v32i16)
11245
0
    return 0;
11246
0
  if ((Subtarget->hasBWI())) {
11247
0
    return fastEmitInst_rr(X86::VPSUBSWZrr, &X86::VR512RegClass, Op0, Op1);
11248
0
  }
11249
0
  return 0;
11250
0
}
11251
11252
0
unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11253
0
  switch (VT.SimpleTy) {
11254
0
  case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
11255
0
  case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
11256
0
  case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
11257
0
  case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
11258
0
  case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
11259
0
  case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
11260
0
  default: return 0;
11261
0
  }
11262
0
}
11263
11264
// FastEmit functions for ISD::STRICT_FADD.
11265
11266
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11267
0
  if (RetVT.SimpleTy != MVT::f16)
11268
0
    return 0;
11269
0
  if ((Subtarget->hasFP16())) {
11270
0
    return fastEmitInst_rr(X86::VADDSHZrr, &X86::FR16XRegClass, Op0, Op1);
11271
0
  }
11272
0
  return 0;
11273
0
}
11274
11275
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11276
0
  if (RetVT.SimpleTy != MVT::f32)
11277
0
    return 0;
11278
0
  if ((Subtarget->hasAVX512())) {
11279
0
    return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op1);
11280
0
  }
11281
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11282
0
    return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op1);
11283
0
  }
11284
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11285
0
    return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op1);
11286
0
  }
11287
0
  if ((!Subtarget->hasSSE1())) {
11288
0
    return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op1);
11289
0
  }
11290
0
  return 0;
11291
0
}
11292
11293
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11294
0
  if (RetVT.SimpleTy != MVT::f64)
11295
0
    return 0;
11296
0
  if ((Subtarget->hasAVX512())) {
11297
0
    return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op1);
11298
0
  }
11299
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11300
0
    return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op1);
11301
0
  }
11302
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11303
0
    return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op1);
11304
0
  }
11305
0
  if ((!Subtarget->hasSSE2())) {
11306
0
    return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op1);
11307
0
  }
11308
0
  return 0;
11309
0
}
11310
11311
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11312
0
  if (RetVT.SimpleTy != MVT::f80)
11313
0
    return 0;
11314
0
  if ((Subtarget->hasX87())) {
11315
0
    return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op1);
11316
0
  }
11317
0
  return 0;
11318
0
}
11319
11320
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11321
0
  if (RetVT.SimpleTy != MVT::v8f16)
11322
0
    return 0;
11323
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11324
0
    return fastEmitInst_rr(X86::VADDPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
11325
0
  }
11326
0
  return 0;
11327
0
}
11328
11329
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11330
0
  if (RetVT.SimpleTy != MVT::v16f16)
11331
0
    return 0;
11332
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11333
0
    return fastEmitInst_rr(X86::VADDPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
11334
0
  }
11335
0
  return 0;
11336
0
}
11337
11338
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11339
0
  if (RetVT.SimpleTy != MVT::v32f16)
11340
0
    return 0;
11341
0
  if ((Subtarget->hasFP16())) {
11342
0
    return fastEmitInst_rr(X86::VADDPHZrr, &X86::VR512RegClass, Op0, Op1);
11343
0
  }
11344
0
  return 0;
11345
0
}
11346
11347
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11348
0
  if (RetVT.SimpleTy != MVT::v4f32)
11349
0
    return 0;
11350
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11351
0
    return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
11352
0
  }
11353
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11354
0
    return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op1);
11355
0
  }
11356
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11357
0
    return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op1);
11358
0
  }
11359
0
  return 0;
11360
0
}
11361
11362
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11363
0
  if (RetVT.SimpleTy != MVT::v8f32)
11364
0
    return 0;
11365
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11366
0
    return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
11367
0
  }
11368
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11369
0
    return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op1);
11370
0
  }
11371
0
  return 0;
11372
0
}
11373
11374
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11375
0
  if (RetVT.SimpleTy != MVT::v16f32)
11376
0
    return 0;
11377
0
  if ((Subtarget->hasAVX512())) {
11378
0
    return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op1);
11379
0
  }
11380
0
  return 0;
11381
0
}
11382
11383
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11384
0
  if (RetVT.SimpleTy != MVT::v2f64)
11385
0
    return 0;
11386
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11387
0
    return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
11388
0
  }
11389
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11390
0
    return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op1);
11391
0
  }
11392
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11393
0
    return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op1);
11394
0
  }
11395
0
  return 0;
11396
0
}
11397
11398
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11399
0
  if (RetVT.SimpleTy != MVT::v4f64)
11400
0
    return 0;
11401
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11402
0
    return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
11403
0
  }
11404
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11405
0
    return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op1);
11406
0
  }
11407
0
  return 0;
11408
0
}
11409
11410
0
unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11411
0
  if (RetVT.SimpleTy != MVT::v8f64)
11412
0
    return 0;
11413
0
  if ((Subtarget->hasAVX512())) {
11414
0
    return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op1);
11415
0
  }
11416
0
  return 0;
11417
0
}
11418
11419
0
unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11420
0
  switch (VT.SimpleTy) {
11421
0
  case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11422
0
  case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11423
0
  case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11424
0
  case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1);
11425
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11426
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1);
11427
0
  case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1);
11428
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11429
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1);
11430
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1);
11431
0
  case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11432
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1);
11433
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1);
11434
0
  default: return 0;
11435
0
  }
11436
0
}
11437
11438
// FastEmit functions for ISD::STRICT_FDIV.
11439
11440
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11441
0
  if (RetVT.SimpleTy != MVT::f16)
11442
0
    return 0;
11443
0
  if ((Subtarget->hasFP16())) {
11444
0
    return fastEmitInst_rr(X86::VDIVSHZrr, &X86::FR16XRegClass, Op0, Op1);
11445
0
  }
11446
0
  return 0;
11447
0
}
11448
11449
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11450
0
  if (RetVT.SimpleTy != MVT::f32)
11451
0
    return 0;
11452
0
  if ((Subtarget->hasAVX512())) {
11453
0
    return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op1);
11454
0
  }
11455
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11456
0
    return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op1);
11457
0
  }
11458
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11459
0
    return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op1);
11460
0
  }
11461
0
  if ((!Subtarget->hasSSE1())) {
11462
0
    return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op1);
11463
0
  }
11464
0
  return 0;
11465
0
}
11466
11467
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11468
0
  if (RetVT.SimpleTy != MVT::f64)
11469
0
    return 0;
11470
0
  if ((Subtarget->hasAVX512())) {
11471
0
    return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op1);
11472
0
  }
11473
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11474
0
    return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op1);
11475
0
  }
11476
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11477
0
    return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op1);
11478
0
  }
11479
0
  if ((!Subtarget->hasSSE2())) {
11480
0
    return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op1);
11481
0
  }
11482
0
  return 0;
11483
0
}
11484
11485
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11486
0
  if (RetVT.SimpleTy != MVT::f80)
11487
0
    return 0;
11488
0
  if ((Subtarget->hasX87())) {
11489
0
    return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op1);
11490
0
  }
11491
0
  return 0;
11492
0
}
11493
11494
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11495
0
  if (RetVT.SimpleTy != MVT::v8f16)
11496
0
    return 0;
11497
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11498
0
    return fastEmitInst_rr(X86::VDIVPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
11499
0
  }
11500
0
  return 0;
11501
0
}
11502
11503
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11504
0
  if (RetVT.SimpleTy != MVT::v16f16)
11505
0
    return 0;
11506
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11507
0
    return fastEmitInst_rr(X86::VDIVPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
11508
0
  }
11509
0
  return 0;
11510
0
}
11511
11512
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11513
0
  if (RetVT.SimpleTy != MVT::v32f16)
11514
0
    return 0;
11515
0
  if ((Subtarget->hasFP16())) {
11516
0
    return fastEmitInst_rr(X86::VDIVPHZrr, &X86::VR512RegClass, Op0, Op1);
11517
0
  }
11518
0
  return 0;
11519
0
}
11520
11521
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11522
0
  if (RetVT.SimpleTy != MVT::v4f32)
11523
0
    return 0;
11524
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11525
0
    return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
11526
0
  }
11527
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11528
0
    return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op1);
11529
0
  }
11530
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11531
0
    return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op1);
11532
0
  }
11533
0
  return 0;
11534
0
}
11535
11536
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11537
0
  if (RetVT.SimpleTy != MVT::v8f32)
11538
0
    return 0;
11539
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11540
0
    return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
11541
0
  }
11542
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11543
0
    return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op1);
11544
0
  }
11545
0
  return 0;
11546
0
}
11547
11548
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11549
0
  if (RetVT.SimpleTy != MVT::v16f32)
11550
0
    return 0;
11551
0
  if ((Subtarget->hasAVX512())) {
11552
0
    return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op1);
11553
0
  }
11554
0
  return 0;
11555
0
}
11556
11557
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11558
0
  if (RetVT.SimpleTy != MVT::v2f64)
11559
0
    return 0;
11560
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11561
0
    return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
11562
0
  }
11563
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11564
0
    return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op1);
11565
0
  }
11566
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11567
0
    return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op1);
11568
0
  }
11569
0
  return 0;
11570
0
}
11571
11572
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11573
0
  if (RetVT.SimpleTy != MVT::v4f64)
11574
0
    return 0;
11575
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11576
0
    return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
11577
0
  }
11578
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11579
0
    return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op1);
11580
0
  }
11581
0
  return 0;
11582
0
}
11583
11584
0
unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11585
0
  if (RetVT.SimpleTy != MVT::v8f64)
11586
0
    return 0;
11587
0
  if ((Subtarget->hasAVX512())) {
11588
0
    return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op1);
11589
0
  }
11590
0
  return 0;
11591
0
}
11592
11593
0
unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11594
0
  switch (VT.SimpleTy) {
11595
0
  case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11596
0
  case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11597
0
  case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11598
0
  case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1);
11599
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11600
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1);
11601
0
  case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1);
11602
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11603
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1);
11604
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1);
11605
0
  case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11606
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1);
11607
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1);
11608
0
  default: return 0;
11609
0
  }
11610
0
}
11611
11612
// FastEmit functions for ISD::STRICT_FMUL.
11613
11614
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11615
0
  if (RetVT.SimpleTy != MVT::f16)
11616
0
    return 0;
11617
0
  if ((Subtarget->hasFP16())) {
11618
0
    return fastEmitInst_rr(X86::VMULSHZrr, &X86::FR16XRegClass, Op0, Op1);
11619
0
  }
11620
0
  return 0;
11621
0
}
11622
11623
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11624
0
  if (RetVT.SimpleTy != MVT::f32)
11625
0
    return 0;
11626
0
  if ((Subtarget->hasAVX512())) {
11627
0
    return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op1);
11628
0
  }
11629
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11630
0
    return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op1);
11631
0
  }
11632
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11633
0
    return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op1);
11634
0
  }
11635
0
  if ((!Subtarget->hasSSE1())) {
11636
0
    return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op1);
11637
0
  }
11638
0
  return 0;
11639
0
}
11640
11641
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11642
0
  if (RetVT.SimpleTy != MVT::f64)
11643
0
    return 0;
11644
0
  if ((Subtarget->hasAVX512())) {
11645
0
    return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op1);
11646
0
  }
11647
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11648
0
    return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op1);
11649
0
  }
11650
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11651
0
    return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op1);
11652
0
  }
11653
0
  if ((!Subtarget->hasSSE2())) {
11654
0
    return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op1);
11655
0
  }
11656
0
  return 0;
11657
0
}
11658
11659
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11660
0
  if (RetVT.SimpleTy != MVT::f80)
11661
0
    return 0;
11662
0
  if ((Subtarget->hasX87())) {
11663
0
    return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op1);
11664
0
  }
11665
0
  return 0;
11666
0
}
11667
11668
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11669
0
  if (RetVT.SimpleTy != MVT::v8f16)
11670
0
    return 0;
11671
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11672
0
    return fastEmitInst_rr(X86::VMULPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
11673
0
  }
11674
0
  return 0;
11675
0
}
11676
11677
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11678
0
  if (RetVT.SimpleTy != MVT::v16f16)
11679
0
    return 0;
11680
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11681
0
    return fastEmitInst_rr(X86::VMULPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
11682
0
  }
11683
0
  return 0;
11684
0
}
11685
11686
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11687
0
  if (RetVT.SimpleTy != MVT::v32f16)
11688
0
    return 0;
11689
0
  if ((Subtarget->hasFP16())) {
11690
0
    return fastEmitInst_rr(X86::VMULPHZrr, &X86::VR512RegClass, Op0, Op1);
11691
0
  }
11692
0
  return 0;
11693
0
}
11694
11695
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11696
0
  if (RetVT.SimpleTy != MVT::v4f32)
11697
0
    return 0;
11698
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11699
0
    return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
11700
0
  }
11701
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11702
0
    return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op1);
11703
0
  }
11704
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11705
0
    return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op1);
11706
0
  }
11707
0
  return 0;
11708
0
}
11709
11710
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11711
0
  if (RetVT.SimpleTy != MVT::v8f32)
11712
0
    return 0;
11713
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11714
0
    return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
11715
0
  }
11716
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11717
0
    return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op1);
11718
0
  }
11719
0
  return 0;
11720
0
}
11721
11722
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11723
0
  if (RetVT.SimpleTy != MVT::v16f32)
11724
0
    return 0;
11725
0
  if ((Subtarget->hasAVX512())) {
11726
0
    return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op1);
11727
0
  }
11728
0
  return 0;
11729
0
}
11730
11731
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11732
0
  if (RetVT.SimpleTy != MVT::v2f64)
11733
0
    return 0;
11734
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11735
0
    return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
11736
0
  }
11737
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11738
0
    return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op1);
11739
0
  }
11740
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11741
0
    return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op1);
11742
0
  }
11743
0
  return 0;
11744
0
}
11745
11746
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11747
0
  if (RetVT.SimpleTy != MVT::v4f64)
11748
0
    return 0;
11749
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11750
0
    return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
11751
0
  }
11752
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11753
0
    return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op1);
11754
0
  }
11755
0
  return 0;
11756
0
}
11757
11758
0
unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11759
0
  if (RetVT.SimpleTy != MVT::v8f64)
11760
0
    return 0;
11761
0
  if ((Subtarget->hasAVX512())) {
11762
0
    return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op1);
11763
0
  }
11764
0
  return 0;
11765
0
}
11766
11767
0
unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11768
0
  switch (VT.SimpleTy) {
11769
0
  case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11770
0
  case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11771
0
  case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11772
0
  case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1);
11773
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11774
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1);
11775
0
  case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1);
11776
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11777
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1);
11778
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1);
11779
0
  case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11780
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1);
11781
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1);
11782
0
  default: return 0;
11783
0
  }
11784
0
}
11785
11786
// FastEmit functions for ISD::STRICT_FSUB.
11787
11788
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11789
0
  if (RetVT.SimpleTy != MVT::f16)
11790
0
    return 0;
11791
0
  if ((Subtarget->hasFP16())) {
11792
0
    return fastEmitInst_rr(X86::VSUBSHZrr, &X86::FR16XRegClass, Op0, Op1);
11793
0
  }
11794
0
  return 0;
11795
0
}
11796
11797
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11798
0
  if (RetVT.SimpleTy != MVT::f32)
11799
0
    return 0;
11800
0
  if ((Subtarget->hasAVX512())) {
11801
0
    return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op1);
11802
0
  }
11803
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11804
0
    return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op1);
11805
0
  }
11806
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11807
0
    return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op1);
11808
0
  }
11809
0
  if ((!Subtarget->hasSSE1())) {
11810
0
    return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op1);
11811
0
  }
11812
0
  return 0;
11813
0
}
11814
11815
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11816
0
  if (RetVT.SimpleTy != MVT::f64)
11817
0
    return 0;
11818
0
  if ((Subtarget->hasAVX512())) {
11819
0
    return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op1);
11820
0
  }
11821
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11822
0
    return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op1);
11823
0
  }
11824
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11825
0
    return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op1);
11826
0
  }
11827
0
  if ((!Subtarget->hasSSE2())) {
11828
0
    return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op1);
11829
0
  }
11830
0
  return 0;
11831
0
}
11832
11833
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11834
0
  if (RetVT.SimpleTy != MVT::f80)
11835
0
    return 0;
11836
0
  if ((Subtarget->hasX87())) {
11837
0
    return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op1);
11838
0
  }
11839
0
  return 0;
11840
0
}
11841
11842
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11843
0
  if (RetVT.SimpleTy != MVT::v8f16)
11844
0
    return 0;
11845
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11846
0
    return fastEmitInst_rr(X86::VSUBPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
11847
0
  }
11848
0
  return 0;
11849
0
}
11850
11851
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11852
0
  if (RetVT.SimpleTy != MVT::v16f16)
11853
0
    return 0;
11854
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
11855
0
    return fastEmitInst_rr(X86::VSUBPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
11856
0
  }
11857
0
  return 0;
11858
0
}
11859
11860
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11861
0
  if (RetVT.SimpleTy != MVT::v32f16)
11862
0
    return 0;
11863
0
  if ((Subtarget->hasFP16())) {
11864
0
    return fastEmitInst_rr(X86::VSUBPHZrr, &X86::VR512RegClass, Op0, Op1);
11865
0
  }
11866
0
  return 0;
11867
0
}
11868
11869
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11870
0
  if (RetVT.SimpleTy != MVT::v4f32)
11871
0
    return 0;
11872
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11873
0
    return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
11874
0
  }
11875
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11876
0
    return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op1);
11877
0
  }
11878
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11879
0
    return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op1);
11880
0
  }
11881
0
  return 0;
11882
0
}
11883
11884
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11885
0
  if (RetVT.SimpleTy != MVT::v8f32)
11886
0
    return 0;
11887
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11888
0
    return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
11889
0
  }
11890
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11891
0
    return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op1);
11892
0
  }
11893
0
  return 0;
11894
0
}
11895
11896
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11897
0
  if (RetVT.SimpleTy != MVT::v16f32)
11898
0
    return 0;
11899
0
  if ((Subtarget->hasAVX512())) {
11900
0
    return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op1);
11901
0
  }
11902
0
  return 0;
11903
0
}
11904
11905
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11906
0
  if (RetVT.SimpleTy != MVT::v2f64)
11907
0
    return 0;
11908
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11909
0
    return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
11910
0
  }
11911
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11912
0
    return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op1);
11913
0
  }
11914
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11915
0
    return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op1);
11916
0
  }
11917
0
  return 0;
11918
0
}
11919
11920
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11921
0
  if (RetVT.SimpleTy != MVT::v4f64)
11922
0
    return 0;
11923
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11924
0
    return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
11925
0
  }
11926
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11927
0
    return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op1);
11928
0
  }
11929
0
  return 0;
11930
0
}
11931
11932
0
unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11933
0
  if (RetVT.SimpleTy != MVT::v8f64)
11934
0
    return 0;
11935
0
  if ((Subtarget->hasAVX512())) {
11936
0
    return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op1);
11937
0
  }
11938
0
  return 0;
11939
0
}
11940
11941
0
unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11942
0
  switch (VT.SimpleTy) {
11943
0
  case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11944
0
  case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11945
0
  case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11946
0
  case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1);
11947
0
  case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11948
0
  case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1);
11949
0
  case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1);
11950
0
  case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11951
0
  case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
11952
0
  case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1);
11953
0
  case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11954
0
  case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
11955
0
  case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1);
11956
0
  default: return 0;
11957
0
  }
11958
0
}
11959
11960
// FastEmit functions for ISD::SUB.
11961
11962
0
unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11963
0
  if (RetVT.SimpleTy != MVT::i8)
11964
0
    return 0;
11965
0
  if ((Subtarget->hasNDD())) {
11966
0
    return fastEmitInst_rr(X86::SUB8rr_ND, &X86::GR8RegClass, Op0, Op1);
11967
0
  }
11968
0
  if ((!Subtarget->hasNDD())) {
11969
0
    return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op1);
11970
0
  }
11971
0
  return 0;
11972
0
}
11973
11974
0
unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11975
0
  if (RetVT.SimpleTy != MVT::i16)
11976
0
    return 0;
11977
0
  if ((Subtarget->hasNDD())) {
11978
0
    return fastEmitInst_rr(X86::SUB16rr_ND, &X86::GR16RegClass, Op0, Op1);
11979
0
  }
11980
0
  if ((!Subtarget->hasNDD())) {
11981
0
    return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op1);
11982
0
  }
11983
0
  return 0;
11984
0
}
11985
11986
0
unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11987
0
  if (RetVT.SimpleTy != MVT::i32)
11988
0
    return 0;
11989
0
  if ((Subtarget->hasNDD())) {
11990
0
    return fastEmitInst_rr(X86::SUB32rr_ND, &X86::GR32RegClass, Op0, Op1);
11991
0
  }
11992
0
  if ((!Subtarget->hasNDD())) {
11993
0
    return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op1);
11994
0
  }
11995
0
  return 0;
11996
0
}
11997
11998
0
unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11999
0
  if (RetVT.SimpleTy != MVT::i64)
12000
0
    return 0;
12001
0
  if ((Subtarget->hasNDD())) {
12002
0
    return fastEmitInst_rr(X86::SUB64rr_ND, &X86::GR64RegClass, Op0, Op1);
12003
0
  }
12004
0
  if ((!Subtarget->hasNDD())) {
12005
0
    return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op1);
12006
0
  }
12007
0
  return 0;
12008
0
}
12009
12010
0
unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12011
0
  if (RetVT.SimpleTy != MVT::v16i8)
12012
0
    return 0;
12013
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12014
0
    return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op1);
12015
0
  }
12016
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12017
0
    return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op1);
12018
0
  }
12019
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12020
0
    return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op1);
12021
0
  }
12022
0
  return 0;
12023
0
}
12024
12025
0
unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12026
0
  if (RetVT.SimpleTy != MVT::v32i8)
12027
0
    return 0;
12028
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12029
0
    return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op1);
12030
0
  }
12031
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12032
0
    return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op1);
12033
0
  }
12034
0
  return 0;
12035
0
}
12036
12037
0
unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12038
0
  if (RetVT.SimpleTy != MVT::v64i8)
12039
0
    return 0;
12040
0
  if ((Subtarget->hasBWI())) {
12041
0
    return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op1);
12042
0
  }
12043
0
  return 0;
12044
0
}
12045
12046
0
unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12047
0
  if (RetVT.SimpleTy != MVT::v8i16)
12048
0
    return 0;
12049
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12050
0
    return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op1);
12051
0
  }
12052
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12053
0
    return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op1);
12054
0
  }
12055
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12056
0
    return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op1);
12057
0
  }
12058
0
  return 0;
12059
0
}
12060
12061
0
unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12062
0
  if (RetVT.SimpleTy != MVT::v16i16)
12063
0
    return 0;
12064
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12065
0
    return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op1);
12066
0
  }
12067
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12068
0
    return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op1);
12069
0
  }
12070
0
  return 0;
12071
0
}
12072
12073
0
unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12074
0
  if (RetVT.SimpleTy != MVT::v32i16)
12075
0
    return 0;
12076
0
  if ((Subtarget->hasBWI())) {
12077
0
    return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op1);
12078
0
  }
12079
0
  return 0;
12080
0
}
12081
12082
0
unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12083
0
  if (RetVT.SimpleTy != MVT::v4i32)
12084
0
    return 0;
12085
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12086
0
    return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op1);
12087
0
  }
12088
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12089
0
    return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op1);
12090
0
  }
12091
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12092
0
    return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op1);
12093
0
  }
12094
0
  return 0;
12095
0
}
12096
12097
0
unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12098
0
  if (RetVT.SimpleTy != MVT::v8i32)
12099
0
    return 0;
12100
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12101
0
    return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op1);
12102
0
  }
12103
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12104
0
    return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op1);
12105
0
  }
12106
0
  return 0;
12107
0
}
12108
12109
0
unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12110
0
  if (RetVT.SimpleTy != MVT::v16i32)
12111
0
    return 0;
12112
0
  if ((Subtarget->hasAVX512())) {
12113
0
    return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op1);
12114
0
  }
12115
0
  return 0;
12116
0
}
12117
12118
0
unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12119
0
  if (RetVT.SimpleTy != MVT::v2i64)
12120
0
    return 0;
12121
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12122
0
    return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12123
0
  }
12124
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12125
0
    return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op1);
12126
0
  }
12127
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12128
0
    return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op1);
12129
0
  }
12130
0
  return 0;
12131
0
}
12132
12133
0
unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12134
0
  if (RetVT.SimpleTy != MVT::v4i64)
12135
0
    return 0;
12136
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12137
0
    return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12138
0
  }
12139
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12140
0
    return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op1);
12141
0
  }
12142
0
  return 0;
12143
0
}
12144
12145
0
unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12146
0
  if (RetVT.SimpleTy != MVT::v8i64)
12147
0
    return 0;
12148
0
  if ((Subtarget->hasAVX512())) {
12149
0
    return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op1);
12150
0
  }
12151
0
  return 0;
12152
0
}
12153
12154
0
unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12155
0
  switch (VT.SimpleTy) {
12156
0
  case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1);
12157
0
  case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1);
12158
0
  case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
12159
0
  case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
12160
0
  case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
12161
0
  case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1);
12162
0
  case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1);
12163
0
  case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
12164
0
  case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1);
12165
0
  case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1);
12166
0
  case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
12167
0
  case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1);
12168
0
  case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1);
12169
0
  case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
12170
0
  case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1);
12171
0
  case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1);
12172
0
  default: return 0;
12173
0
  }
12174
0
}
12175
12176
// FastEmit functions for ISD::UADDSAT.
12177
12178
0
unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12179
0
  if (RetVT.SimpleTy != MVT::v16i8)
12180
0
    return 0;
12181
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12182
0
    return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
12183
0
  }
12184
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12185
0
    return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op1);
12186
0
  }
12187
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12188
0
    return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op1);
12189
0
  }
12190
0
  return 0;
12191
0
}
12192
12193
0
unsigned fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12194
0
  if (RetVT.SimpleTy != MVT::v32i8)
12195
0
    return 0;
12196
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12197
0
    return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
12198
0
  }
12199
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12200
0
    return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op1);
12201
0
  }
12202
0
  return 0;
12203
0
}
12204
12205
0
unsigned fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12206
0
  if (RetVT.SimpleTy != MVT::v64i8)
12207
0
    return 0;
12208
0
  if ((Subtarget->hasBWI())) {
12209
0
    return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op1);
12210
0
  }
12211
0
  return 0;
12212
0
}
12213
12214
0
unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12215
0
  if (RetVT.SimpleTy != MVT::v8i16)
12216
0
    return 0;
12217
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12218
0
    return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
12219
0
  }
12220
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12221
0
    return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op1);
12222
0
  }
12223
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12224
0
    return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op1);
12225
0
  }
12226
0
  return 0;
12227
0
}
12228
12229
0
unsigned fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12230
0
  if (RetVT.SimpleTy != MVT::v16i16)
12231
0
    return 0;
12232
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12233
0
    return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
12234
0
  }
12235
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12236
0
    return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op1);
12237
0
  }
12238
0
  return 0;
12239
0
}
12240
12241
0
unsigned fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12242
0
  if (RetVT.SimpleTy != MVT::v32i16)
12243
0
    return 0;
12244
0
  if ((Subtarget->hasBWI())) {
12245
0
    return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op1);
12246
0
  }
12247
0
  return 0;
12248
0
}
12249
12250
0
unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12251
0
  switch (VT.SimpleTy) {
12252
0
  case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12253
0
  case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12254
0
  case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12255
0
  case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12256
0
  case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12257
0
  case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12258
0
  default: return 0;
12259
0
  }
12260
0
}
12261
12262
// FastEmit functions for ISD::UMAX.
12263
12264
0
unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12265
0
  if (RetVT.SimpleTy != MVT::v16i8)
12266
0
    return 0;
12267
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12268
0
    return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op1);
12269
0
  }
12270
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12271
0
    return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op1);
12272
0
  }
12273
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12274
0
    return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op1);
12275
0
  }
12276
0
  return 0;
12277
0
}
12278
12279
0
unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12280
0
  if (RetVT.SimpleTy != MVT::v32i8)
12281
0
    return 0;
12282
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12283
0
    return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op1);
12284
0
  }
12285
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12286
0
    return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op1);
12287
0
  }
12288
0
  return 0;
12289
0
}
12290
12291
0
unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12292
0
  if (RetVT.SimpleTy != MVT::v64i8)
12293
0
    return 0;
12294
0
  if ((Subtarget->hasBWI())) {
12295
0
    return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op1);
12296
0
  }
12297
0
  return 0;
12298
0
}
12299
12300
0
unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12301
0
  if (RetVT.SimpleTy != MVT::v8i16)
12302
0
    return 0;
12303
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12304
0
    return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op1);
12305
0
  }
12306
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12307
0
    return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op1);
12308
0
  }
12309
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12310
0
    return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op1);
12311
0
  }
12312
0
  return 0;
12313
0
}
12314
12315
0
unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12316
0
  if (RetVT.SimpleTy != MVT::v16i16)
12317
0
    return 0;
12318
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12319
0
    return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op1);
12320
0
  }
12321
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12322
0
    return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op1);
12323
0
  }
12324
0
  return 0;
12325
0
}
12326
12327
0
unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12328
0
  if (RetVT.SimpleTy != MVT::v32i16)
12329
0
    return 0;
12330
0
  if ((Subtarget->hasBWI())) {
12331
0
    return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op1);
12332
0
  }
12333
0
  return 0;
12334
0
}
12335
12336
0
unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12337
0
  if (RetVT.SimpleTy != MVT::v4i32)
12338
0
    return 0;
12339
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12340
0
    return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op1);
12341
0
  }
12342
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12343
0
    return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op1);
12344
0
  }
12345
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12346
0
    return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op1);
12347
0
  }
12348
0
  return 0;
12349
0
}
12350
12351
0
unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12352
0
  if (RetVT.SimpleTy != MVT::v8i32)
12353
0
    return 0;
12354
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12355
0
    return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op1);
12356
0
  }
12357
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12358
0
    return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op1);
12359
0
  }
12360
0
  return 0;
12361
0
}
12362
12363
0
unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12364
0
  if (RetVT.SimpleTy != MVT::v16i32)
12365
0
    return 0;
12366
0
  if ((Subtarget->hasAVX512())) {
12367
0
    return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op1);
12368
0
  }
12369
0
  return 0;
12370
0
}
12371
12372
0
unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12373
0
  if (RetVT.SimpleTy != MVT::v2i64)
12374
0
    return 0;
12375
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12376
0
    return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12377
0
  }
12378
0
  return 0;
12379
0
}
12380
12381
0
unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12382
0
  if (RetVT.SimpleTy != MVT::v4i64)
12383
0
    return 0;
12384
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12385
0
    return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12386
0
  }
12387
0
  return 0;
12388
0
}
12389
12390
0
unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12391
0
  if (RetVT.SimpleTy != MVT::v8i64)
12392
0
    return 0;
12393
0
  if ((Subtarget->hasAVX512())) {
12394
0
    return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op1);
12395
0
  }
12396
0
  return 0;
12397
0
}
12398
12399
0
unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12400
0
  switch (VT.SimpleTy) {
12401
0
  case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12402
0
  case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1);
12403
0
  case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1);
12404
0
  case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12405
0
  case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1);
12406
0
  case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1);
12407
0
  case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12408
0
  case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1);
12409
0
  case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1);
12410
0
  case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1);
12411
0
  case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1);
12412
0
  case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1);
12413
0
  default: return 0;
12414
0
  }
12415
0
}
12416
12417
// FastEmit functions for ISD::UMIN.
12418
12419
0
unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12420
0
  if (RetVT.SimpleTy != MVT::v16i8)
12421
0
    return 0;
12422
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12423
0
    return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op1);
12424
0
  }
12425
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12426
0
    return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op1);
12427
0
  }
12428
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12429
0
    return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op1);
12430
0
  }
12431
0
  return 0;
12432
0
}
12433
12434
0
unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12435
0
  if (RetVT.SimpleTy != MVT::v32i8)
12436
0
    return 0;
12437
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12438
0
    return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op1);
12439
0
  }
12440
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12441
0
    return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op1);
12442
0
  }
12443
0
  return 0;
12444
0
}
12445
12446
0
unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12447
0
  if (RetVT.SimpleTy != MVT::v64i8)
12448
0
    return 0;
12449
0
  if ((Subtarget->hasBWI())) {
12450
0
    return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op1);
12451
0
  }
12452
0
  return 0;
12453
0
}
12454
12455
0
unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12456
0
  if (RetVT.SimpleTy != MVT::v8i16)
12457
0
    return 0;
12458
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12459
0
    return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op1);
12460
0
  }
12461
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12462
0
    return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op1);
12463
0
  }
12464
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12465
0
    return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op1);
12466
0
  }
12467
0
  return 0;
12468
0
}
12469
12470
0
unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12471
0
  if (RetVT.SimpleTy != MVT::v16i16)
12472
0
    return 0;
12473
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12474
0
    return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op1);
12475
0
  }
12476
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12477
0
    return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op1);
12478
0
  }
12479
0
  return 0;
12480
0
}
12481
12482
0
unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12483
0
  if (RetVT.SimpleTy != MVT::v32i16)
12484
0
    return 0;
12485
0
  if ((Subtarget->hasBWI())) {
12486
0
    return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op1);
12487
0
  }
12488
0
  return 0;
12489
0
}
12490
12491
0
unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12492
0
  if (RetVT.SimpleTy != MVT::v4i32)
12493
0
    return 0;
12494
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12495
0
    return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op1);
12496
0
  }
12497
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12498
0
    return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op1);
12499
0
  }
12500
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12501
0
    return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op1);
12502
0
  }
12503
0
  return 0;
12504
0
}
12505
12506
0
unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12507
0
  if (RetVT.SimpleTy != MVT::v8i32)
12508
0
    return 0;
12509
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12510
0
    return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op1);
12511
0
  }
12512
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12513
0
    return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op1);
12514
0
  }
12515
0
  return 0;
12516
0
}
12517
12518
0
unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12519
0
  if (RetVT.SimpleTy != MVT::v16i32)
12520
0
    return 0;
12521
0
  if ((Subtarget->hasAVX512())) {
12522
0
    return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op1);
12523
0
  }
12524
0
  return 0;
12525
0
}
12526
12527
0
unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12528
0
  if (RetVT.SimpleTy != MVT::v2i64)
12529
0
    return 0;
12530
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12531
0
    return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12532
0
  }
12533
0
  return 0;
12534
0
}
12535
12536
0
unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12537
0
  if (RetVT.SimpleTy != MVT::v4i64)
12538
0
    return 0;
12539
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12540
0
    return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12541
0
  }
12542
0
  return 0;
12543
0
}
12544
12545
0
unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12546
0
  if (RetVT.SimpleTy != MVT::v8i64)
12547
0
    return 0;
12548
0
  if ((Subtarget->hasAVX512())) {
12549
0
    return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op1);
12550
0
  }
12551
0
  return 0;
12552
0
}
12553
12554
0
unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12555
0
  switch (VT.SimpleTy) {
12556
0
  case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12557
0
  case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1);
12558
0
  case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1);
12559
0
  case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12560
0
  case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1);
12561
0
  case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1);
12562
0
  case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12563
0
  case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1);
12564
0
  case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1);
12565
0
  case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1);
12566
0
  case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1);
12567
0
  case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1);
12568
0
  default: return 0;
12569
0
  }
12570
0
}
12571
12572
// FastEmit functions for ISD::USUBSAT.
12573
12574
0
unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12575
0
  if (RetVT.SimpleTy != MVT::v16i8)
12576
0
    return 0;
12577
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12578
0
    return fastEmitInst_rr(X86::VPSUBUSBZ128rr, &X86::VR128XRegClass, Op0, Op1);
12579
0
  }
12580
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12581
0
    return fastEmitInst_rr(X86::PSUBUSBrr, &X86::VR128RegClass, Op0, Op1);
12582
0
  }
12583
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12584
0
    return fastEmitInst_rr(X86::VPSUBUSBrr, &X86::VR128RegClass, Op0, Op1);
12585
0
  }
12586
0
  return 0;
12587
0
}
12588
12589
0
unsigned fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12590
0
  if (RetVT.SimpleTy != MVT::v32i8)
12591
0
    return 0;
12592
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12593
0
    return fastEmitInst_rr(X86::VPSUBUSBZ256rr, &X86::VR256XRegClass, Op0, Op1);
12594
0
  }
12595
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12596
0
    return fastEmitInst_rr(X86::VPSUBUSBYrr, &X86::VR256RegClass, Op0, Op1);
12597
0
  }
12598
0
  return 0;
12599
0
}
12600
12601
0
unsigned fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12602
0
  if (RetVT.SimpleTy != MVT::v64i8)
12603
0
    return 0;
12604
0
  if ((Subtarget->hasBWI())) {
12605
0
    return fastEmitInst_rr(X86::VPSUBUSBZrr, &X86::VR512RegClass, Op0, Op1);
12606
0
  }
12607
0
  return 0;
12608
0
}
12609
12610
0
unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12611
0
  if (RetVT.SimpleTy != MVT::v8i16)
12612
0
    return 0;
12613
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12614
0
    return fastEmitInst_rr(X86::VPSUBUSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
12615
0
  }
12616
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12617
0
    return fastEmitInst_rr(X86::PSUBUSWrr, &X86::VR128RegClass, Op0, Op1);
12618
0
  }
12619
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12620
0
    return fastEmitInst_rr(X86::VPSUBUSWrr, &X86::VR128RegClass, Op0, Op1);
12621
0
  }
12622
0
  return 0;
12623
0
}
12624
12625
0
unsigned fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12626
0
  if (RetVT.SimpleTy != MVT::v16i16)
12627
0
    return 0;
12628
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12629
0
    return fastEmitInst_rr(X86::VPSUBUSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
12630
0
  }
12631
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12632
0
    return fastEmitInst_rr(X86::VPSUBUSWYrr, &X86::VR256RegClass, Op0, Op1);
12633
0
  }
12634
0
  return 0;
12635
0
}
12636
12637
0
unsigned fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12638
0
  if (RetVT.SimpleTy != MVT::v32i16)
12639
0
    return 0;
12640
0
  if ((Subtarget->hasBWI())) {
12641
0
    return fastEmitInst_rr(X86::VPSUBUSWZrr, &X86::VR512RegClass, Op0, Op1);
12642
0
  }
12643
0
  return 0;
12644
0
}
12645
12646
0
unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12647
0
  switch (VT.SimpleTy) {
12648
0
  case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12649
0
  case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1);
12650
0
  case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1);
12651
0
  case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12652
0
  case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1);
12653
0
  case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1);
12654
0
  default: return 0;
12655
0
  }
12656
0
}
12657
12658
// FastEmit functions for ISD::XOR.
12659
12660
0
unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12661
0
  if (RetVT.SimpleTy != MVT::i8)
12662
0
    return 0;
12663
0
  if ((Subtarget->hasNDD())) {
12664
0
    return fastEmitInst_rr(X86::XOR8rr_ND, &X86::GR8RegClass, Op0, Op1);
12665
0
  }
12666
0
  if ((!Subtarget->hasNDD())) {
12667
0
    return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op1);
12668
0
  }
12669
0
  return 0;
12670
0
}
12671
12672
0
unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12673
0
  if (RetVT.SimpleTy != MVT::i16)
12674
0
    return 0;
12675
0
  if ((Subtarget->hasNDD())) {
12676
0
    return fastEmitInst_rr(X86::XOR16rr_ND, &X86::GR16RegClass, Op0, Op1);
12677
0
  }
12678
0
  if ((!Subtarget->hasNDD())) {
12679
0
    return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op1);
12680
0
  }
12681
0
  return 0;
12682
0
}
12683
12684
0
unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12685
0
  if (RetVT.SimpleTy != MVT::i32)
12686
0
    return 0;
12687
0
  if ((Subtarget->hasNDD())) {
12688
0
    return fastEmitInst_rr(X86::XOR32rr_ND, &X86::GR32RegClass, Op0, Op1);
12689
0
  }
12690
0
  if ((!Subtarget->hasNDD())) {
12691
0
    return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op1);
12692
0
  }
12693
0
  return 0;
12694
0
}
12695
12696
0
unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12697
0
  if (RetVT.SimpleTy != MVT::i64)
12698
0
    return 0;
12699
0
  if ((Subtarget->hasNDD())) {
12700
0
    return fastEmitInst_rr(X86::XOR64rr_ND, &X86::GR64RegClass, Op0, Op1);
12701
0
  }
12702
0
  if ((!Subtarget->hasNDD())) {
12703
0
    return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op1);
12704
0
  }
12705
0
  return 0;
12706
0
}
12707
12708
0
unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12709
0
  if (RetVT.SimpleTy != MVT::v8i1)
12710
0
    return 0;
12711
0
  if ((Subtarget->hasDQI())) {
12712
0
    return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op1);
12713
0
  }
12714
0
  return 0;
12715
0
}
12716
12717
0
unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12718
0
  if (RetVT.SimpleTy != MVT::v16i1)
12719
0
    return 0;
12720
0
  if ((Subtarget->hasAVX512())) {
12721
0
    return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op1);
12722
0
  }
12723
0
  return 0;
12724
0
}
12725
12726
0
unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12727
0
  if (RetVT.SimpleTy != MVT::v32i1)
12728
0
    return 0;
12729
0
  if ((Subtarget->hasBWI())) {
12730
0
    return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op1);
12731
0
  }
12732
0
  return 0;
12733
0
}
12734
12735
0
unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12736
0
  if (RetVT.SimpleTy != MVT::v64i1)
12737
0
    return 0;
12738
0
  if ((Subtarget->hasBWI())) {
12739
0
    return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op1);
12740
0
  }
12741
0
  return 0;
12742
0
}
12743
12744
0
unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12745
0
  if (RetVT.SimpleTy != MVT::v16i8)
12746
0
    return 0;
12747
0
  if ((Subtarget->hasVLX())) {
12748
0
    return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12749
0
  }
12750
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12751
0
    return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1);
12752
0
  }
12753
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12754
0
    return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1);
12755
0
  }
12756
0
  return 0;
12757
0
}
12758
12759
0
unsigned fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12760
0
  if (RetVT.SimpleTy != MVT::v32i8)
12761
0
    return 0;
12762
0
  if ((Subtarget->hasVLX())) {
12763
0
    return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12764
0
  }
12765
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12766
0
    return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1);
12767
0
  }
12768
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12769
0
    return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1);
12770
0
  }
12771
0
  return 0;
12772
0
}
12773
12774
0
unsigned fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12775
0
  if (RetVT.SimpleTy != MVT::v64i8)
12776
0
    return 0;
12777
0
  if ((Subtarget->hasAVX512())) {
12778
0
    return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1);
12779
0
  }
12780
0
  return 0;
12781
0
}
12782
12783
0
unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12784
0
  if (RetVT.SimpleTy != MVT::v8i16)
12785
0
    return 0;
12786
0
  if ((Subtarget->hasVLX())) {
12787
0
    return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12788
0
  }
12789
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12790
0
    return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1);
12791
0
  }
12792
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12793
0
    return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1);
12794
0
  }
12795
0
  return 0;
12796
0
}
12797
12798
0
unsigned fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12799
0
  if (RetVT.SimpleTy != MVT::v16i16)
12800
0
    return 0;
12801
0
  if ((Subtarget->hasVLX())) {
12802
0
    return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12803
0
  }
12804
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12805
0
    return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1);
12806
0
  }
12807
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12808
0
    return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1);
12809
0
  }
12810
0
  return 0;
12811
0
}
12812
12813
0
unsigned fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12814
0
  if (RetVT.SimpleTy != MVT::v32i16)
12815
0
    return 0;
12816
0
  if ((Subtarget->hasAVX512())) {
12817
0
    return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1);
12818
0
  }
12819
0
  return 0;
12820
0
}
12821
12822
0
unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12823
0
  if (RetVT.SimpleTy != MVT::v4i32)
12824
0
    return 0;
12825
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12826
0
    return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1);
12827
0
  }
12828
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12829
0
    return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1);
12830
0
  }
12831
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12832
0
    return fastEmitInst_rr(X86::VPXORDZ128rr, &X86::VR128XRegClass, Op0, Op1);
12833
0
  }
12834
0
  return 0;
12835
0
}
12836
12837
0
unsigned fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12838
0
  if (RetVT.SimpleTy != MVT::v8i32)
12839
0
    return 0;
12840
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12841
0
    return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1);
12842
0
  }
12843
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12844
0
    return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1);
12845
0
  }
12846
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12847
0
    return fastEmitInst_rr(X86::VPXORDZ256rr, &X86::VR256XRegClass, Op0, Op1);
12848
0
  }
12849
0
  return 0;
12850
0
}
12851
12852
0
unsigned fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12853
0
  if (RetVT.SimpleTy != MVT::v16i32)
12854
0
    return 0;
12855
0
  if ((Subtarget->hasAVX512())) {
12856
0
    return fastEmitInst_rr(X86::VPXORDZrr, &X86::VR512RegClass, Op0, Op1);
12857
0
  }
12858
0
  return 0;
12859
0
}
12860
12861
0
unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12862
0
  if (RetVT.SimpleTy != MVT::v2i64)
12863
0
    return 0;
12864
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12865
0
    return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12866
0
  }
12867
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12868
0
    return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1);
12869
0
  }
12870
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12871
0
    return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1);
12872
0
  }
12873
0
  return 0;
12874
0
}
12875
12876
0
unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12877
0
  if (RetVT.SimpleTy != MVT::v4i64)
12878
0
    return 0;
12879
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12880
0
    return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1);
12881
0
  }
12882
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12883
0
    return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1);
12884
0
  }
12885
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12886
0
    return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1);
12887
0
  }
12888
0
  return 0;
12889
0
}
12890
12891
0
unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12892
0
  if (RetVT.SimpleTy != MVT::v8i64)
12893
0
    return 0;
12894
0
  if ((Subtarget->hasAVX512())) {
12895
0
    return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1);
12896
0
  }
12897
0
  return 0;
12898
0
}
12899
12900
0
unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12901
0
  switch (VT.SimpleTy) {
12902
0
  case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1);
12903
0
  case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1);
12904
0
  case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
12905
0
  case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
12906
0
  case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1);
12907
0
  case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1);
12908
0
  case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1);
12909
0
  case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1);
12910
0
  case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
12911
0
  case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1);
12912
0
  case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1);
12913
0
  case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
12914
0
  case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1);
12915
0
  case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1);
12916
0
  case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
12917
0
  case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1);
12918
0
  case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1);
12919
0
  case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
12920
0
  case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1);
12921
0
  case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1);
12922
0
  default: return 0;
12923
0
  }
12924
0
}
12925
12926
// FastEmit functions for X86ISD::ADDSUB.
12927
12928
0
unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12929
0
  if (RetVT.SimpleTy != MVT::v4f32)
12930
0
    return 0;
12931
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
12932
0
    return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op1);
12933
0
  }
12934
0
  if ((Subtarget->hasAVX())) {
12935
0
    return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op1);
12936
0
  }
12937
0
  return 0;
12938
0
}
12939
12940
0
unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12941
0
  if (RetVT.SimpleTy != MVT::v8f32)
12942
0
    return 0;
12943
0
  if ((Subtarget->hasAVX())) {
12944
0
    return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op1);
12945
0
  }
12946
0
  return 0;
12947
0
}
12948
12949
0
unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12950
0
  if (RetVT.SimpleTy != MVT::v2f64)
12951
0
    return 0;
12952
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
12953
0
    return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op1);
12954
0
  }
12955
0
  if ((Subtarget->hasAVX())) {
12956
0
    return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op1);
12957
0
  }
12958
0
  return 0;
12959
0
}
12960
12961
0
unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12962
0
  if (RetVT.SimpleTy != MVT::v4f64)
12963
0
    return 0;
12964
0
  if ((Subtarget->hasAVX())) {
12965
0
    return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op1);
12966
0
  }
12967
0
  return 0;
12968
0
}
12969
12970
0
unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12971
0
  switch (VT.SimpleTy) {
12972
0
  case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
12973
0
  case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
12974
0
  case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
12975
0
  case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
12976
0
  default: return 0;
12977
0
  }
12978
0
}
12979
12980
// FastEmit functions for X86ISD::ANDNP.
12981
12982
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12983
0
  if (RetVT.SimpleTy != MVT::v16i8)
12984
0
    return 0;
12985
0
  if ((Subtarget->hasVLX())) {
12986
0
    return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1);
12987
0
  }
12988
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12989
0
    return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1);
12990
0
  }
12991
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12992
0
    return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1);
12993
0
  }
12994
0
  return 0;
12995
0
}
12996
12997
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12998
0
  if (RetVT.SimpleTy != MVT::v32i8)
12999
0
    return 0;
13000
0
  if ((Subtarget->hasVLX())) {
13001
0
    return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1);
13002
0
  }
13003
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13004
0
    return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1);
13005
0
  }
13006
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13007
0
    return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1);
13008
0
  }
13009
0
  return 0;
13010
0
}
13011
13012
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13013
0
  if (RetVT.SimpleTy != MVT::v64i8)
13014
0
    return 0;
13015
0
  if ((Subtarget->hasAVX512())) {
13016
0
    return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1);
13017
0
  }
13018
0
  return 0;
13019
0
}
13020
13021
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13022
0
  if (RetVT.SimpleTy != MVT::v8i16)
13023
0
    return 0;
13024
0
  if ((Subtarget->hasVLX())) {
13025
0
    return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1);
13026
0
  }
13027
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13028
0
    return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1);
13029
0
  }
13030
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13031
0
    return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1);
13032
0
  }
13033
0
  return 0;
13034
0
}
13035
13036
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13037
0
  if (RetVT.SimpleTy != MVT::v16i16)
13038
0
    return 0;
13039
0
  if ((Subtarget->hasVLX())) {
13040
0
    return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1);
13041
0
  }
13042
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13043
0
    return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1);
13044
0
  }
13045
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13046
0
    return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1);
13047
0
  }
13048
0
  return 0;
13049
0
}
13050
13051
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13052
0
  if (RetVT.SimpleTy != MVT::v32i16)
13053
0
    return 0;
13054
0
  if ((Subtarget->hasAVX512())) {
13055
0
    return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1);
13056
0
  }
13057
0
  return 0;
13058
0
}
13059
13060
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13061
0
  if (RetVT.SimpleTy != MVT::v4i32)
13062
0
    return 0;
13063
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13064
0
    return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1);
13065
0
  }
13066
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13067
0
    return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1);
13068
0
  }
13069
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13070
0
    return fastEmitInst_rr(X86::VPANDNDZ128rr, &X86::VR128XRegClass, Op0, Op1);
13071
0
  }
13072
0
  return 0;
13073
0
}
13074
13075
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13076
0
  if (RetVT.SimpleTy != MVT::v8i32)
13077
0
    return 0;
13078
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13079
0
    return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1);
13080
0
  }
13081
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13082
0
    return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1);
13083
0
  }
13084
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13085
0
    return fastEmitInst_rr(X86::VPANDNDZ256rr, &X86::VR256XRegClass, Op0, Op1);
13086
0
  }
13087
0
  return 0;
13088
0
}
13089
13090
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13091
0
  if (RetVT.SimpleTy != MVT::v16i32)
13092
0
    return 0;
13093
0
  if ((Subtarget->hasAVX512())) {
13094
0
    return fastEmitInst_rr(X86::VPANDNDZrr, &X86::VR512RegClass, Op0, Op1);
13095
0
  }
13096
0
  return 0;
13097
0
}
13098
13099
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13100
0
  if (RetVT.SimpleTy != MVT::v2i64)
13101
0
    return 0;
13102
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13103
0
    return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1);
13104
0
  }
13105
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13106
0
    return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1);
13107
0
  }
13108
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13109
0
    return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1);
13110
0
  }
13111
0
  return 0;
13112
0
}
13113
13114
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13115
0
  if (RetVT.SimpleTy != MVT::v4i64)
13116
0
    return 0;
13117
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13118
0
    return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1);
13119
0
  }
13120
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13121
0
    return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1);
13122
0
  }
13123
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13124
0
    return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1);
13125
0
  }
13126
0
  return 0;
13127
0
}
13128
13129
0
unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13130
0
  if (RetVT.SimpleTy != MVT::v8i64)
13131
0
    return 0;
13132
0
  if ((Subtarget->hasAVX512())) {
13133
0
    return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1);
13134
0
  }
13135
0
  return 0;
13136
0
}
13137
13138
0
unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13139
0
  switch (VT.SimpleTy) {
13140
0
  case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1);
13141
0
  case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1);
13142
0
  case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1);
13143
0
  case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1);
13144
0
  case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1);
13145
0
  case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1);
13146
0
  case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1);
13147
0
  case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1);
13148
0
  case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1);
13149
0
  case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1);
13150
0
  case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1);
13151
0
  case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1);
13152
0
  default: return 0;
13153
0
  }
13154
0
}
13155
13156
// FastEmit functions for X86ISD::BEXTR.
13157
13158
0
unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13159
0
  if (RetVT.SimpleTy != MVT::i32)
13160
0
    return 0;
13161
0
  if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13162
0
    return fastEmitInst_rr(X86::BEXTR32rr_EVEX, &X86::GR32RegClass, Op0, Op1);
13163
0
  }
13164
0
  if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
13165
0
    return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op1);
13166
0
  }
13167
0
  return 0;
13168
0
}
13169
13170
0
unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13171
0
  if (RetVT.SimpleTy != MVT::i64)
13172
0
    return 0;
13173
0
  if ((Subtarget->hasBMI()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13174
0
    return fastEmitInst_rr(X86::BEXTR64rr_EVEX, &X86::GR64RegClass, Op0, Op1);
13175
0
  }
13176
0
  if ((Subtarget->hasBMI()) && (!Subtarget->hasEGPR())) {
13177
0
    return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op1);
13178
0
  }
13179
0
  return 0;
13180
0
}
13181
13182
0
unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13183
0
  switch (VT.SimpleTy) {
13184
0
  case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1);
13185
0
  case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1);
13186
0
  default: return 0;
13187
0
  }
13188
0
}
13189
13190
// FastEmit functions for X86ISD::BT.
13191
13192
0
unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13193
0
  if (RetVT.SimpleTy != MVT::i32)
13194
0
    return 0;
13195
0
  return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op1);
13196
0
}
13197
13198
0
unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13199
0
  if (RetVT.SimpleTy != MVT::i32)
13200
0
    return 0;
13201
0
  return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op1);
13202
0
}
13203
13204
0
unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13205
0
  if (RetVT.SimpleTy != MVT::i32)
13206
0
    return 0;
13207
0
  return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op1);
13208
0
}
13209
13210
0
unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13211
0
  switch (VT.SimpleTy) {
13212
0
  case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1);
13213
0
  case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1);
13214
0
  case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1);
13215
0
  default: return 0;
13216
0
  }
13217
0
}
13218
13219
// FastEmit functions for X86ISD::BZHI.
13220
13221
0
unsigned fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13222
0
  if (RetVT.SimpleTy != MVT::i32)
13223
0
    return 0;
13224
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13225
0
    return fastEmitInst_rr(X86::BZHI32rr_EVEX, &X86::GR32RegClass, Op0, Op1);
13226
0
  }
13227
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
13228
0
    return fastEmitInst_rr(X86::BZHI32rr, &X86::GR32RegClass, Op0, Op1);
13229
0
  }
13230
0
  return 0;
13231
0
}
13232
13233
0
unsigned fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13234
0
  if (RetVT.SimpleTy != MVT::i64)
13235
0
    return 0;
13236
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR()) && (Subtarget->is64Bit())) {
13237
0
    return fastEmitInst_rr(X86::BZHI64rr_EVEX, &X86::GR64RegClass, Op0, Op1);
13238
0
  }
13239
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
13240
0
    return fastEmitInst_rr(X86::BZHI64rr, &X86::GR64RegClass, Op0, Op1);
13241
0
  }
13242
0
  return 0;
13243
0
}
13244
13245
0
unsigned fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13246
0
  switch (VT.SimpleTy) {
13247
0
  case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1);
13248
0
  case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1);
13249
0
  default: return 0;
13250
0
  }
13251
0
}
13252
13253
// FastEmit functions for X86ISD::CMP.
13254
13255
0
unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13256
0
  if (RetVT.SimpleTy != MVT::i32)
13257
0
    return 0;
13258
0
  return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op1);
13259
0
}
13260
13261
0
unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13262
0
  if (RetVT.SimpleTy != MVT::i32)
13263
0
    return 0;
13264
0
  return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op1);
13265
0
}
13266
13267
0
unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13268
0
  if (RetVT.SimpleTy != MVT::i32)
13269
0
    return 0;
13270
0
  return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op1);
13271
0
}
13272
13273
0
unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13274
0
  if (RetVT.SimpleTy != MVT::i32)
13275
0
    return 0;
13276
0
  return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op1);
13277
0
}
13278
13279
0
unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13280
0
  switch (VT.SimpleTy) {
13281
0
  case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1);
13282
0
  case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1);
13283
0
  case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
13284
0
  case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1);
13285
0
  default: return 0;
13286
0
  }
13287
0
}
13288
13289
// FastEmit functions for X86ISD::COMI.
13290
13291
0
unsigned fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13292
0
  if (RetVT.SimpleTy != MVT::i32)
13293
0
    return 0;
13294
0
  if ((Subtarget->hasFP16())) {
13295
0
    return fastEmitInst_rr(X86::VCOMISHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13296
0
  }
13297
0
  return 0;
13298
0
}
13299
13300
0
unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13301
0
  if (RetVT.SimpleTy != MVT::i32)
13302
0
    return 0;
13303
0
  if ((Subtarget->hasAVX512())) {
13304
0
    return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13305
0
  }
13306
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13307
0
    return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op1);
13308
0
  }
13309
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13310
0
    return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1);
13311
0
  }
13312
0
  return 0;
13313
0
}
13314
13315
0
unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13316
0
  if (RetVT.SimpleTy != MVT::i32)
13317
0
    return 0;
13318
0
  if ((Subtarget->hasAVX512())) {
13319
0
    return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13320
0
  }
13321
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13322
0
    return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op1);
13323
0
  }
13324
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13325
0
    return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1);
13326
0
  }
13327
0
  return 0;
13328
0
}
13329
13330
0
unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13331
0
  switch (VT.SimpleTy) {
13332
0
  case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1);
13333
0
  case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1);
13334
0
  case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1);
13335
0
  default: return 0;
13336
0
  }
13337
0
}
13338
13339
// FastEmit functions for X86ISD::CVTNE2PS2BF16.
13340
13341
0
unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13342
0
  if (RetVT.SimpleTy != MVT::v8bf16)
13343
0
    return 0;
13344
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
13345
0
    return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op1);
13346
0
  }
13347
0
  return 0;
13348
0
}
13349
13350
0
unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13351
0
  if (RetVT.SimpleTy != MVT::v16bf16)
13352
0
    return 0;
13353
0
  if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
13354
0
    return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z256rr, &X86::VR256XRegClass, Op0, Op1);
13355
0
  }
13356
0
  return 0;
13357
0
}
13358
13359
0
unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13360
0
  if (RetVT.SimpleTy != MVT::v32bf16)
13361
0
    return 0;
13362
0
  if ((Subtarget->hasBF16())) {
13363
0
    return fastEmitInst_rr(X86::VCVTNE2PS2BF16Zrr, &X86::VR512RegClass, Op0, Op1);
13364
0
  }
13365
0
  return 0;
13366
0
}
13367
13368
0
unsigned fastEmit_X86ISD_CVTNE2PS2BF16_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13369
0
  switch (VT.SimpleTy) {
13370
0
  case MVT::v4f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(RetVT, Op0, Op1);
13371
0
  case MVT::v8f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(RetVT, Op0, Op1);
13372
0
  case MVT::v16f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(RetVT, Op0, Op1);
13373
0
  default: return 0;
13374
0
  }
13375
0
}
13376
13377
// FastEmit functions for X86ISD::FADDS.
13378
13379
0
unsigned fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13380
0
  if (RetVT.SimpleTy != MVT::v8f16)
13381
0
    return 0;
13382
0
  if ((Subtarget->hasFP16())) {
13383
0
    return fastEmitInst_rr(X86::VADDSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13384
0
  }
13385
0
  return 0;
13386
0
}
13387
13388
0
unsigned fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13389
0
  if (RetVT.SimpleTy != MVT::v4f32)
13390
0
    return 0;
13391
0
  if ((Subtarget->hasAVX512())) {
13392
0
    return fastEmitInst_rr(X86::VADDSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13393
0
  }
13394
0
  return 0;
13395
0
}
13396
13397
0
unsigned fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13398
0
  if (RetVT.SimpleTy != MVT::v2f64)
13399
0
    return 0;
13400
0
  if ((Subtarget->hasAVX512())) {
13401
0
    return fastEmitInst_rr(X86::VADDSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13402
0
  }
13403
0
  return 0;
13404
0
}
13405
13406
0
unsigned fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13407
0
  switch (VT.SimpleTy) {
13408
0
  case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1);
13409
0
  case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1);
13410
0
  case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1);
13411
0
  default: return 0;
13412
0
  }
13413
0
}
13414
13415
// FastEmit functions for X86ISD::FAND.
13416
13417
0
unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13418
0
  if (RetVT.SimpleTy != MVT::f128)
13419
0
    return 0;
13420
0
  if ((Subtarget->hasVLX())) {
13421
0
    return fastEmitInst_rr(X86::VANDPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
13422
0
  }
13423
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13424
0
    return fastEmitInst_rr(X86::VANDPSrr, &X86::VR128RegClass, Op0, Op1);
13425
0
  }
13426
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13427
0
    return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op1);
13428
0
  }
13429
0
  return 0;
13430
0
}
13431
13432
0
unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13433
0
  if (RetVT.SimpleTy != MVT::v4f32)
13434
0
    return 0;
13435
0
  return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op1);
13436
0
}
13437
13438
0
unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13439
0
  switch (VT.SimpleTy) {
13440
0
  case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1);
13441
0
  case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1);
13442
0
  default: return 0;
13443
0
  }
13444
0
}
13445
13446
// FastEmit functions for X86ISD::FANDN.
13447
13448
0
unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13449
0
  if (RetVT.SimpleTy != MVT::v4f32)
13450
0
    return 0;
13451
0
  return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op1);
13452
0
}
13453
13454
0
unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13455
0
  switch (VT.SimpleTy) {
13456
0
  case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1);
13457
0
  default: return 0;
13458
0
  }
13459
0
}
13460
13461
// FastEmit functions for X86ISD::FCMP.
13462
13463
0
unsigned fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13464
0
  if (RetVT.SimpleTy != MVT::i32)
13465
0
    return 0;
13466
0
  if ((Subtarget->hasFP16())) {
13467
0
    return fastEmitInst_rr(X86::VUCOMISHZrr, &X86::FR16XRegClass, Op0, Op1);
13468
0
  }
13469
0
  return 0;
13470
0
}
13471
13472
0
unsigned fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13473
0
  if (RetVT.SimpleTy != MVT::i32)
13474
0
    return 0;
13475
0
  if ((Subtarget->hasAVX512())) {
13476
0
    return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op1);
13477
0
  }
13478
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13479
0
    return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op1);
13480
0
  }
13481
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13482
0
    return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op1);
13483
0
  }
13484
0
  if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
13485
0
    return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op1);
13486
0
  }
13487
0
  return 0;
13488
0
}
13489
13490
0
unsigned fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13491
0
  if (RetVT.SimpleTy != MVT::i32)
13492
0
    return 0;
13493
0
  if ((Subtarget->hasAVX512())) {
13494
0
    return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op1);
13495
0
  }
13496
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13497
0
    return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op1);
13498
0
  }
13499
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13500
0
    return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op1);
13501
0
  }
13502
0
  if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
13503
0
    return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op1);
13504
0
  }
13505
0
  return 0;
13506
0
}
13507
13508
0
unsigned fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13509
0
  if (RetVT.SimpleTy != MVT::i32)
13510
0
    return 0;
13511
0
  if ((Subtarget->canUseCMOV())) {
13512
0
    return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op1);
13513
0
  }
13514
0
  return 0;
13515
0
}
13516
13517
0
unsigned fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13518
0
  switch (VT.SimpleTy) {
13519
0
  case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
13520
0
  case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
13521
0
  case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
13522
0
  case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
13523
0
  default: return 0;
13524
0
  }
13525
0
}
13526
13527
// FastEmit functions for X86ISD::FDIVS.
13528
13529
0
unsigned fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13530
0
  if (RetVT.SimpleTy != MVT::v8f16)
13531
0
    return 0;
13532
0
  if ((Subtarget->hasFP16())) {
13533
0
    return fastEmitInst_rr(X86::VDIVSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13534
0
  }
13535
0
  return 0;
13536
0
}
13537
13538
0
unsigned fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13539
0
  if (RetVT.SimpleTy != MVT::v4f32)
13540
0
    return 0;
13541
0
  if ((Subtarget->hasAVX512())) {
13542
0
    return fastEmitInst_rr(X86::VDIVSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13543
0
  }
13544
0
  return 0;
13545
0
}
13546
13547
0
unsigned fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13548
0
  if (RetVT.SimpleTy != MVT::v2f64)
13549
0
    return 0;
13550
0
  if ((Subtarget->hasAVX512())) {
13551
0
    return fastEmitInst_rr(X86::VDIVSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
13552
0
  }
13553
0
  return 0;
13554
0
}
13555
13556
0
unsigned fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13557
0
  switch (VT.SimpleTy) {
13558
0
  case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1);
13559
0
  case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1);
13560
0
  case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1);
13561
0
  default: return 0;
13562
0
  }
13563
0
}
13564
13565
// FastEmit functions for X86ISD::FGETEXPS.
13566
13567
0
unsigned fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13568
0
  if (RetVT.SimpleTy != MVT::v8f16)
13569
0
    return 0;
13570
0
  if ((Subtarget->hasFP16())) {
13571
0
    return fastEmitInst_rr(X86::VGETEXPSHZr, &X86::VR128XRegClass, Op0, Op1);
13572
0
  }
13573
0
  return 0;
13574
0
}
13575
13576
0
unsigned fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13577
0
  if (RetVT.SimpleTy != MVT::v4f32)
13578
0
    return 0;
13579
0
  if ((Subtarget->hasAVX512())) {
13580
0
    return fastEmitInst_rr(X86::VGETEXPSSZr, &X86::VR128XRegClass, Op0, Op1);
13581
0
  }
13582
0
  return 0;
13583
0
}
13584
13585
0
unsigned fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13586
0
  if (RetVT.SimpleTy != MVT::v2f64)
13587
0
    return 0;
13588
0
  if ((Subtarget->hasAVX512())) {
13589
0
    return fastEmitInst_rr(X86::VGETEXPSDZr, &X86::VR128XRegClass, Op0, Op1);
13590
0
  }
13591
0
  return 0;
13592
0
}
13593
13594
0
unsigned fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13595
0
  switch (VT.SimpleTy) {
13596
0
  case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1);
13597
0
  case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1);
13598
0
  case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1);
13599
0
  default: return 0;
13600
0
  }
13601
0
}
13602
13603
// FastEmit functions for X86ISD::FGETEXPS_SAE.
13604
13605
0
unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13606
0
  if (RetVT.SimpleTy != MVT::v8f16)
13607
0
    return 0;
13608
0
  if ((Subtarget->hasFP16())) {
13609
0
    return fastEmitInst_rr(X86::VGETEXPSHZrb, &X86::VR128XRegClass, Op0, Op1);
13610
0
  }
13611
0
  return 0;
13612
0
}
13613
13614
0
unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13615
0
  if (RetVT.SimpleTy != MVT::v4f32)
13616
0
    return 0;
13617
0
  if ((Subtarget->hasAVX512())) {
13618
0
    return fastEmitInst_rr(X86::VGETEXPSSZrb, &X86::VR128XRegClass, Op0, Op1);
13619
0
  }
13620
0
  return 0;
13621
0
}
13622
13623
0
unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13624
0
  if (RetVT.SimpleTy != MVT::v2f64)
13625
0
    return 0;
13626
0
  if ((Subtarget->hasAVX512())) {
13627
0
    return fastEmitInst_rr(X86::VGETEXPSDZrb, &X86::VR128XRegClass, Op0, Op1);
13628
0
  }
13629
0
  return 0;
13630
0
}
13631
13632
0
unsigned fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13633
0
  switch (VT.SimpleTy) {
13634
0
  case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
13635
0
  case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
13636
0
  case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
13637
0
  default: return 0;
13638
0
  }
13639
0
}
13640
13641
// FastEmit functions for X86ISD::FHADD.
13642
13643
0
unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13644
0
  if (RetVT.SimpleTy != MVT::v4f32)
13645
0
    return 0;
13646
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13647
0
    return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op1);
13648
0
  }
13649
0
  if ((Subtarget->hasAVX())) {
13650
0
    return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op1);
13651
0
  }
13652
0
  return 0;
13653
0
}
13654
13655
0
unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13656
0
  if (RetVT.SimpleTy != MVT::v8f32)
13657
0
    return 0;
13658
0
  if ((Subtarget->hasAVX())) {
13659
0
    return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op1);
13660
0
  }
13661
0
  return 0;
13662
0
}
13663
13664
0
unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13665
0
  if (RetVT.SimpleTy != MVT::v2f64)
13666
0
    return 0;
13667
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13668
0
    return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op1);
13669
0
  }
13670
0
  if ((Subtarget->hasAVX())) {
13671
0
    return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op1);
13672
0
  }
13673
0
  return 0;
13674
0
}
13675
13676
0
unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13677
0
  if (RetVT.SimpleTy != MVT::v4f64)
13678
0
    return 0;
13679
0
  if ((Subtarget->hasAVX())) {
13680
0
    return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op1);
13681
0
  }
13682
0
  return 0;
13683
0
}
13684
13685
0
unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13686
0
  switch (VT.SimpleTy) {
13687
0
  case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1);
13688
0
  case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1);
13689
0
  case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1);
13690
0
  case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1);
13691
0
  default: return 0;
13692
0
  }
13693
0
}
13694
13695
// FastEmit functions for X86ISD::FHSUB.
13696
13697
0
unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13698
0
  if (RetVT.SimpleTy != MVT::v4f32)
13699
0
    return 0;
13700
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13701
0
    return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op1);
13702
0
  }
13703
0
  if ((Subtarget->hasAVX())) {
13704
0
    return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op1);
13705
0
  }
13706
0
  return 0;
13707
0
}
13708
13709
0
unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13710
0
  if (RetVT.SimpleTy != MVT::v8f32)
13711
0
    return 0;
13712
0
  if ((Subtarget->hasAVX())) {
13713
0
    return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op1);
13714
0
  }
13715
0
  return 0;
13716
0
}
13717
13718
0
unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13719
0
  if (RetVT.SimpleTy != MVT::v2f64)
13720
0
    return 0;
13721
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
13722
0
    return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op1);
13723
0
  }
13724
0
  if ((Subtarget->hasAVX())) {
13725
0
    return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op1);
13726
0
  }
13727
0
  return 0;
13728
0
}
13729
13730
0
unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13731
0
  if (RetVT.SimpleTy != MVT::v4f64)
13732
0
    return 0;
13733
0
  if ((Subtarget->hasAVX())) {
13734
0
    return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op1);
13735
0
  }
13736
0
  return 0;
13737
0
}
13738
13739
0
unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13740
0
  switch (VT.SimpleTy) {
13741
0
  case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13742
0
  case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1);
13743
0
  case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13744
0
  case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1);
13745
0
  default: return 0;
13746
0
  }
13747
0
}
13748
13749
// FastEmit functions for X86ISD::FMAX.
13750
13751
0
unsigned fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13752
0
  if (RetVT.SimpleTy != MVT::f16)
13753
0
    return 0;
13754
0
  if ((Subtarget->hasFP16())) {
13755
0
    return fastEmitInst_rr(X86::VMAXSHZrr, &X86::FR16XRegClass, Op0, Op1);
13756
0
  }
13757
0
  return 0;
13758
0
}
13759
13760
0
unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13761
0
  if (RetVT.SimpleTy != MVT::f32)
13762
0
    return 0;
13763
0
  if ((Subtarget->hasAVX512())) {
13764
0
    return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op1);
13765
0
  }
13766
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13767
0
    return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op1);
13768
0
  }
13769
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13770
0
    return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op1);
13771
0
  }
13772
0
  return 0;
13773
0
}
13774
13775
0
unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13776
0
  if (RetVT.SimpleTy != MVT::f64)
13777
0
    return 0;
13778
0
  if ((Subtarget->hasAVX512())) {
13779
0
    return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op1);
13780
0
  }
13781
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13782
0
    return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op1);
13783
0
  }
13784
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13785
0
    return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op1);
13786
0
  }
13787
0
  return 0;
13788
0
}
13789
13790
0
unsigned fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13791
0
  if (RetVT.SimpleTy != MVT::v8f16)
13792
0
    return 0;
13793
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13794
0
    return fastEmitInst_rr(X86::VMAXPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
13795
0
  }
13796
0
  return 0;
13797
0
}
13798
13799
0
unsigned fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13800
0
  if (RetVT.SimpleTy != MVT::v16f16)
13801
0
    return 0;
13802
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13803
0
    return fastEmitInst_rr(X86::VMAXPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
13804
0
  }
13805
0
  return 0;
13806
0
}
13807
13808
0
unsigned fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13809
0
  if (RetVT.SimpleTy != MVT::v32f16)
13810
0
    return 0;
13811
0
  if ((Subtarget->hasFP16())) {
13812
0
    return fastEmitInst_rr(X86::VMAXPHZrr, &X86::VR512RegClass, Op0, Op1);
13813
0
  }
13814
0
  return 0;
13815
0
}
13816
13817
0
unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13818
0
  if (RetVT.SimpleTy != MVT::v4f32)
13819
0
    return 0;
13820
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13821
0
    return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
13822
0
  }
13823
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13824
0
    return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op1);
13825
0
  }
13826
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13827
0
    return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op1);
13828
0
  }
13829
0
  return 0;
13830
0
}
13831
13832
0
unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13833
0
  if (RetVT.SimpleTy != MVT::v8f32)
13834
0
    return 0;
13835
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13836
0
    return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
13837
0
  }
13838
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13839
0
    return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op1);
13840
0
  }
13841
0
  return 0;
13842
0
}
13843
13844
0
unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13845
0
  if (RetVT.SimpleTy != MVT::v16f32)
13846
0
    return 0;
13847
0
  if ((Subtarget->hasAVX512())) {
13848
0
    return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op1);
13849
0
  }
13850
0
  return 0;
13851
0
}
13852
13853
0
unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13854
0
  if (RetVT.SimpleTy != MVT::v2f64)
13855
0
    return 0;
13856
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13857
0
    return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
13858
0
  }
13859
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13860
0
    return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op1);
13861
0
  }
13862
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13863
0
    return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op1);
13864
0
  }
13865
0
  return 0;
13866
0
}
13867
13868
0
unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13869
0
  if (RetVT.SimpleTy != MVT::v4f64)
13870
0
    return 0;
13871
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13872
0
    return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
13873
0
  }
13874
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13875
0
    return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op1);
13876
0
  }
13877
0
  return 0;
13878
0
}
13879
13880
0
unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13881
0
  if (RetVT.SimpleTy != MVT::v8f64)
13882
0
    return 0;
13883
0
  if ((Subtarget->hasAVX512())) {
13884
0
    return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op1);
13885
0
  }
13886
0
  return 0;
13887
0
}
13888
13889
0
unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13890
0
  switch (VT.SimpleTy) {
13891
0
  case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1);
13892
0
  case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1);
13893
0
  case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1);
13894
0
  case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1);
13895
0
  case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1);
13896
0
  case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1);
13897
0
  case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1);
13898
0
  case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1);
13899
0
  case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1);
13900
0
  case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1);
13901
0
  case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1);
13902
0
  case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1);
13903
0
  default: return 0;
13904
0
  }
13905
0
}
13906
13907
// FastEmit functions for X86ISD::FMAXC.
13908
13909
0
unsigned fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13910
0
  if (RetVT.SimpleTy != MVT::f16)
13911
0
    return 0;
13912
0
  if ((Subtarget->hasAVX512())) {
13913
0
    return fastEmitInst_rr(X86::VMAXCSHZrr, &X86::FR16XRegClass, Op0, Op1);
13914
0
  }
13915
0
  return 0;
13916
0
}
13917
13918
0
unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13919
0
  if (RetVT.SimpleTy != MVT::f32)
13920
0
    return 0;
13921
0
  if ((Subtarget->hasAVX512())) {
13922
0
    return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op1);
13923
0
  }
13924
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13925
0
    return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op1);
13926
0
  }
13927
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13928
0
    return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op1);
13929
0
  }
13930
0
  return 0;
13931
0
}
13932
13933
0
unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13934
0
  if (RetVT.SimpleTy != MVT::f64)
13935
0
    return 0;
13936
0
  if ((Subtarget->hasAVX512())) {
13937
0
    return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op1);
13938
0
  }
13939
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13940
0
    return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op1);
13941
0
  }
13942
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13943
0
    return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op1);
13944
0
  }
13945
0
  return 0;
13946
0
}
13947
13948
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13949
0
  if (RetVT.SimpleTy != MVT::v8f16)
13950
0
    return 0;
13951
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13952
0
    return fastEmitInst_rr(X86::VMAXCPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
13953
0
  }
13954
0
  return 0;
13955
0
}
13956
13957
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13958
0
  if (RetVT.SimpleTy != MVT::v16f16)
13959
0
    return 0;
13960
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
13961
0
    return fastEmitInst_rr(X86::VMAXCPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
13962
0
  }
13963
0
  return 0;
13964
0
}
13965
13966
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13967
0
  if (RetVT.SimpleTy != MVT::v32f16)
13968
0
    return 0;
13969
0
  if ((Subtarget->hasFP16())) {
13970
0
    return fastEmitInst_rr(X86::VMAXCPHZrr, &X86::VR512RegClass, Op0, Op1);
13971
0
  }
13972
0
  return 0;
13973
0
}
13974
13975
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13976
0
  if (RetVT.SimpleTy != MVT::v4f32)
13977
0
    return 0;
13978
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13979
0
    return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
13980
0
  }
13981
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13982
0
    return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op1);
13983
0
  }
13984
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13985
0
    return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op1);
13986
0
  }
13987
0
  return 0;
13988
0
}
13989
13990
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13991
0
  if (RetVT.SimpleTy != MVT::v8f32)
13992
0
    return 0;
13993
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13994
0
    return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
13995
0
  }
13996
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13997
0
    return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op1);
13998
0
  }
13999
0
  return 0;
14000
0
}
14001
14002
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14003
0
  if (RetVT.SimpleTy != MVT::v16f32)
14004
0
    return 0;
14005
0
  if ((Subtarget->hasAVX512())) {
14006
0
    return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op1);
14007
0
  }
14008
0
  return 0;
14009
0
}
14010
14011
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14012
0
  if (RetVT.SimpleTy != MVT::v2f64)
14013
0
    return 0;
14014
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14015
0
    return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
14016
0
  }
14017
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14018
0
    return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op1);
14019
0
  }
14020
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14021
0
    return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op1);
14022
0
  }
14023
0
  return 0;
14024
0
}
14025
14026
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14027
0
  if (RetVT.SimpleTy != MVT::v4f64)
14028
0
    return 0;
14029
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14030
0
    return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
14031
0
  }
14032
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14033
0
    return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op1);
14034
0
  }
14035
0
  return 0;
14036
0
}
14037
14038
0
unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14039
0
  if (RetVT.SimpleTy != MVT::v8f64)
14040
0
    return 0;
14041
0
  if ((Subtarget->hasAVX512())) {
14042
0
    return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op1);
14043
0
  }
14044
0
  return 0;
14045
0
}
14046
14047
0
unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14048
0
  switch (VT.SimpleTy) {
14049
0
  case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1);
14050
0
  case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1);
14051
0
  case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1);
14052
0
  case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1);
14053
0
  case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1);
14054
0
  case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1);
14055
0
  case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1);
14056
0
  case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1);
14057
0
  case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1);
14058
0
  case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1);
14059
0
  case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1);
14060
0
  case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1);
14061
0
  default: return 0;
14062
0
  }
14063
0
}
14064
14065
// FastEmit functions for X86ISD::FMAXS.
14066
14067
0
unsigned fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14068
0
  if (RetVT.SimpleTy != MVT::v8f16)
14069
0
    return 0;
14070
0
  if ((Subtarget->hasFP16())) {
14071
0
    return fastEmitInst_rr(X86::VMAXSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14072
0
  }
14073
0
  return 0;
14074
0
}
14075
14076
0
unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14077
0
  if (RetVT.SimpleTy != MVT::v4f32)
14078
0
    return 0;
14079
0
  if ((Subtarget->hasAVX512())) {
14080
0
    return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14081
0
  }
14082
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14083
0
    return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op1);
14084
0
  }
14085
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14086
0
    return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op1);
14087
0
  }
14088
0
  return 0;
14089
0
}
14090
14091
0
unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14092
0
  if (RetVT.SimpleTy != MVT::v2f64)
14093
0
    return 0;
14094
0
  if ((Subtarget->hasAVX512())) {
14095
0
    return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14096
0
  }
14097
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14098
0
    return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op1);
14099
0
  }
14100
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14101
0
    return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op1);
14102
0
  }
14103
0
  return 0;
14104
0
}
14105
14106
0
unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14107
0
  switch (VT.SimpleTy) {
14108
0
  case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1);
14109
0
  case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1);
14110
0
  case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1);
14111
0
  default: return 0;
14112
0
  }
14113
0
}
14114
14115
// FastEmit functions for X86ISD::FMAXS_SAE.
14116
14117
0
unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14118
0
  if (RetVT.SimpleTy != MVT::v8f16)
14119
0
    return 0;
14120
0
  if ((Subtarget->hasFP16())) {
14121
0
    return fastEmitInst_rr(X86::VMAXSHZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14122
0
  }
14123
0
  return 0;
14124
0
}
14125
14126
0
unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14127
0
  if (RetVT.SimpleTy != MVT::v4f32)
14128
0
    return 0;
14129
0
  if ((Subtarget->hasAVX512())) {
14130
0
    return fastEmitInst_rr(X86::VMAXSSZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14131
0
  }
14132
0
  return 0;
14133
0
}
14134
14135
0
unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14136
0
  if (RetVT.SimpleTy != MVT::v2f64)
14137
0
    return 0;
14138
0
  if ((Subtarget->hasAVX512())) {
14139
0
    return fastEmitInst_rr(X86::VMAXSDZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14140
0
  }
14141
0
  return 0;
14142
0
}
14143
14144
0
unsigned fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14145
0
  switch (VT.SimpleTy) {
14146
0
  case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
14147
0
  case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
14148
0
  case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
14149
0
  default: return 0;
14150
0
  }
14151
0
}
14152
14153
// FastEmit functions for X86ISD::FMAX_SAE.
14154
14155
0
unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14156
0
  if (RetVT.SimpleTy != MVT::v32f16)
14157
0
    return 0;
14158
0
  if ((Subtarget->hasFP16())) {
14159
0
    return fastEmitInst_rr(X86::VMAXPHZrrb, &X86::VR512RegClass, Op0, Op1);
14160
0
  }
14161
0
  return 0;
14162
0
}
14163
14164
0
unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14165
0
  if (RetVT.SimpleTy != MVT::v16f32)
14166
0
    return 0;
14167
0
  if ((Subtarget->hasAVX512())) {
14168
0
    return fastEmitInst_rr(X86::VMAXPSZrrb, &X86::VR512RegClass, Op0, Op1);
14169
0
  }
14170
0
  return 0;
14171
0
}
14172
14173
0
unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14174
0
  if (RetVT.SimpleTy != MVT::v8f64)
14175
0
    return 0;
14176
0
  if ((Subtarget->hasAVX512())) {
14177
0
    return fastEmitInst_rr(X86::VMAXPDZrrb, &X86::VR512RegClass, Op0, Op1);
14178
0
  }
14179
0
  return 0;
14180
0
}
14181
14182
0
unsigned fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14183
0
  switch (VT.SimpleTy) {
14184
0
  case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
14185
0
  case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
14186
0
  case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
14187
0
  default: return 0;
14188
0
  }
14189
0
}
14190
14191
// FastEmit functions for X86ISD::FMIN.
14192
14193
0
unsigned fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14194
0
  if (RetVT.SimpleTy != MVT::f16)
14195
0
    return 0;
14196
0
  if ((Subtarget->hasFP16())) {
14197
0
    return fastEmitInst_rr(X86::VMINSHZrr, &X86::FR16XRegClass, Op0, Op1);
14198
0
  }
14199
0
  return 0;
14200
0
}
14201
14202
0
unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14203
0
  if (RetVT.SimpleTy != MVT::f32)
14204
0
    return 0;
14205
0
  if ((Subtarget->hasAVX512())) {
14206
0
    return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op1);
14207
0
  }
14208
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14209
0
    return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op1);
14210
0
  }
14211
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14212
0
    return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op1);
14213
0
  }
14214
0
  return 0;
14215
0
}
14216
14217
0
unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14218
0
  if (RetVT.SimpleTy != MVT::f64)
14219
0
    return 0;
14220
0
  if ((Subtarget->hasAVX512())) {
14221
0
    return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op1);
14222
0
  }
14223
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14224
0
    return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op1);
14225
0
  }
14226
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14227
0
    return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op1);
14228
0
  }
14229
0
  return 0;
14230
0
}
14231
14232
0
unsigned fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14233
0
  if (RetVT.SimpleTy != MVT::v8f16)
14234
0
    return 0;
14235
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14236
0
    return fastEmitInst_rr(X86::VMINPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
14237
0
  }
14238
0
  return 0;
14239
0
}
14240
14241
0
unsigned fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14242
0
  if (RetVT.SimpleTy != MVT::v16f16)
14243
0
    return 0;
14244
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14245
0
    return fastEmitInst_rr(X86::VMINPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
14246
0
  }
14247
0
  return 0;
14248
0
}
14249
14250
0
unsigned fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14251
0
  if (RetVT.SimpleTy != MVT::v32f16)
14252
0
    return 0;
14253
0
  if ((Subtarget->hasFP16())) {
14254
0
    return fastEmitInst_rr(X86::VMINPHZrr, &X86::VR512RegClass, Op0, Op1);
14255
0
  }
14256
0
  return 0;
14257
0
}
14258
14259
0
unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14260
0
  if (RetVT.SimpleTy != MVT::v4f32)
14261
0
    return 0;
14262
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14263
0
    return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
14264
0
  }
14265
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14266
0
    return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op1);
14267
0
  }
14268
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14269
0
    return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op1);
14270
0
  }
14271
0
  return 0;
14272
0
}
14273
14274
0
unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14275
0
  if (RetVT.SimpleTy != MVT::v8f32)
14276
0
    return 0;
14277
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14278
0
    return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
14279
0
  }
14280
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14281
0
    return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op1);
14282
0
  }
14283
0
  return 0;
14284
0
}
14285
14286
0
unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14287
0
  if (RetVT.SimpleTy != MVT::v16f32)
14288
0
    return 0;
14289
0
  if ((Subtarget->hasAVX512())) {
14290
0
    return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op1);
14291
0
  }
14292
0
  return 0;
14293
0
}
14294
14295
0
unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14296
0
  if (RetVT.SimpleTy != MVT::v2f64)
14297
0
    return 0;
14298
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14299
0
    return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
14300
0
  }
14301
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14302
0
    return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op1);
14303
0
  }
14304
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14305
0
    return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op1);
14306
0
  }
14307
0
  return 0;
14308
0
}
14309
14310
0
unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14311
0
  if (RetVT.SimpleTy != MVT::v4f64)
14312
0
    return 0;
14313
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14314
0
    return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
14315
0
  }
14316
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14317
0
    return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op1);
14318
0
  }
14319
0
  return 0;
14320
0
}
14321
14322
0
unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14323
0
  if (RetVT.SimpleTy != MVT::v8f64)
14324
0
    return 0;
14325
0
  if ((Subtarget->hasAVX512())) {
14326
0
    return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op1);
14327
0
  }
14328
0
  return 0;
14329
0
}
14330
14331
0
unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14332
0
  switch (VT.SimpleTy) {
14333
0
  case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1);
14334
0
  case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1);
14335
0
  case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1);
14336
0
  case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1);
14337
0
  case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1);
14338
0
  case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1);
14339
0
  case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1);
14340
0
  case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1);
14341
0
  case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1);
14342
0
  case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1);
14343
0
  case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1);
14344
0
  case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1);
14345
0
  default: return 0;
14346
0
  }
14347
0
}
14348
14349
// FastEmit functions for X86ISD::FMINC.
14350
14351
0
unsigned fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14352
0
  if (RetVT.SimpleTy != MVT::f16)
14353
0
    return 0;
14354
0
  if ((Subtarget->hasAVX512())) {
14355
0
    return fastEmitInst_rr(X86::VMINCSHZrr, &X86::FR16XRegClass, Op0, Op1);
14356
0
  }
14357
0
  return 0;
14358
0
}
14359
14360
0
unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14361
0
  if (RetVT.SimpleTy != MVT::f32)
14362
0
    return 0;
14363
0
  if ((Subtarget->hasAVX512())) {
14364
0
    return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op1);
14365
0
  }
14366
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14367
0
    return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op1);
14368
0
  }
14369
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14370
0
    return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op1);
14371
0
  }
14372
0
  return 0;
14373
0
}
14374
14375
0
unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14376
0
  if (RetVT.SimpleTy != MVT::f64)
14377
0
    return 0;
14378
0
  if ((Subtarget->hasAVX512())) {
14379
0
    return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op1);
14380
0
  }
14381
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14382
0
    return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op1);
14383
0
  }
14384
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14385
0
    return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op1);
14386
0
  }
14387
0
  return 0;
14388
0
}
14389
14390
0
unsigned fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14391
0
  if (RetVT.SimpleTy != MVT::v8f16)
14392
0
    return 0;
14393
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14394
0
    return fastEmitInst_rr(X86::VMINCPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
14395
0
  }
14396
0
  return 0;
14397
0
}
14398
14399
0
unsigned fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14400
0
  if (RetVT.SimpleTy != MVT::v16f16)
14401
0
    return 0;
14402
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
14403
0
    return fastEmitInst_rr(X86::VMINCPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
14404
0
  }
14405
0
  return 0;
14406
0
}
14407
14408
0
unsigned fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14409
0
  if (RetVT.SimpleTy != MVT::v32f16)
14410
0
    return 0;
14411
0
  if ((Subtarget->hasFP16())) {
14412
0
    return fastEmitInst_rr(X86::VMINCPHZrr, &X86::VR512RegClass, Op0, Op1);
14413
0
  }
14414
0
  return 0;
14415
0
}
14416
14417
0
unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14418
0
  if (RetVT.SimpleTy != MVT::v4f32)
14419
0
    return 0;
14420
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14421
0
    return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
14422
0
  }
14423
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14424
0
    return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op1);
14425
0
  }
14426
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14427
0
    return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op1);
14428
0
  }
14429
0
  return 0;
14430
0
}
14431
14432
0
unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14433
0
  if (RetVT.SimpleTy != MVT::v8f32)
14434
0
    return 0;
14435
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14436
0
    return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
14437
0
  }
14438
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14439
0
    return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op1);
14440
0
  }
14441
0
  return 0;
14442
0
}
14443
14444
0
unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14445
0
  if (RetVT.SimpleTy != MVT::v16f32)
14446
0
    return 0;
14447
0
  if ((Subtarget->hasAVX512())) {
14448
0
    return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op1);
14449
0
  }
14450
0
  return 0;
14451
0
}
14452
14453
0
unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14454
0
  if (RetVT.SimpleTy != MVT::v2f64)
14455
0
    return 0;
14456
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14457
0
    return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
14458
0
  }
14459
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14460
0
    return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op1);
14461
0
  }
14462
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14463
0
    return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op1);
14464
0
  }
14465
0
  return 0;
14466
0
}
14467
14468
0
unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14469
0
  if (RetVT.SimpleTy != MVT::v4f64)
14470
0
    return 0;
14471
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14472
0
    return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
14473
0
  }
14474
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14475
0
    return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op1);
14476
0
  }
14477
0
  return 0;
14478
0
}
14479
14480
0
unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14481
0
  if (RetVT.SimpleTy != MVT::v8f64)
14482
0
    return 0;
14483
0
  if ((Subtarget->hasAVX512())) {
14484
0
    return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op1);
14485
0
  }
14486
0
  return 0;
14487
0
}
14488
14489
0
unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14490
0
  switch (VT.SimpleTy) {
14491
0
  case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1);
14492
0
  case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1);
14493
0
  case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1);
14494
0
  case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1);
14495
0
  case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1);
14496
0
  case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1);
14497
0
  case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1);
14498
0
  case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1);
14499
0
  case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1);
14500
0
  case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1);
14501
0
  case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1);
14502
0
  case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1);
14503
0
  default: return 0;
14504
0
  }
14505
0
}
14506
14507
// FastEmit functions for X86ISD::FMINS.
14508
14509
0
unsigned fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14510
0
  if (RetVT.SimpleTy != MVT::v8f16)
14511
0
    return 0;
14512
0
  if ((Subtarget->hasFP16())) {
14513
0
    return fastEmitInst_rr(X86::VMINSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14514
0
  }
14515
0
  return 0;
14516
0
}
14517
14518
0
unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14519
0
  if (RetVT.SimpleTy != MVT::v4f32)
14520
0
    return 0;
14521
0
  if ((Subtarget->hasAVX512())) {
14522
0
    return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14523
0
  }
14524
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14525
0
    return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op1);
14526
0
  }
14527
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14528
0
    return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op1);
14529
0
  }
14530
0
  return 0;
14531
0
}
14532
14533
0
unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14534
0
  if (RetVT.SimpleTy != MVT::v2f64)
14535
0
    return 0;
14536
0
  if ((Subtarget->hasAVX512())) {
14537
0
    return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14538
0
  }
14539
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14540
0
    return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op1);
14541
0
  }
14542
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
14543
0
    return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op1);
14544
0
  }
14545
0
  return 0;
14546
0
}
14547
14548
0
unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14549
0
  switch (VT.SimpleTy) {
14550
0
  case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1);
14551
0
  case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1);
14552
0
  case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1);
14553
0
  default: return 0;
14554
0
  }
14555
0
}
14556
14557
// FastEmit functions for X86ISD::FMINS_SAE.
14558
14559
0
unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14560
0
  if (RetVT.SimpleTy != MVT::v8f16)
14561
0
    return 0;
14562
0
  if ((Subtarget->hasFP16())) {
14563
0
    return fastEmitInst_rr(X86::VMINSHZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14564
0
  }
14565
0
  return 0;
14566
0
}
14567
14568
0
unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14569
0
  if (RetVT.SimpleTy != MVT::v4f32)
14570
0
    return 0;
14571
0
  if ((Subtarget->hasAVX512())) {
14572
0
    return fastEmitInst_rr(X86::VMINSSZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14573
0
  }
14574
0
  return 0;
14575
0
}
14576
14577
0
unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14578
0
  if (RetVT.SimpleTy != MVT::v2f64)
14579
0
    return 0;
14580
0
  if ((Subtarget->hasAVX512())) {
14581
0
    return fastEmitInst_rr(X86::VMINSDZrrb_Int, &X86::VR128XRegClass, Op0, Op1);
14582
0
  }
14583
0
  return 0;
14584
0
}
14585
14586
0
unsigned fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14587
0
  switch (VT.SimpleTy) {
14588
0
  case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1);
14589
0
  case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
14590
0
  case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
14591
0
  default: return 0;
14592
0
  }
14593
0
}
14594
14595
// FastEmit functions for X86ISD::FMIN_SAE.
14596
14597
0
unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14598
0
  if (RetVT.SimpleTy != MVT::v32f16)
14599
0
    return 0;
14600
0
  if ((Subtarget->hasFP16())) {
14601
0
    return fastEmitInst_rr(X86::VMINPHZrrb, &X86::VR512RegClass, Op0, Op1);
14602
0
  }
14603
0
  return 0;
14604
0
}
14605
14606
0
unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14607
0
  if (RetVT.SimpleTy != MVT::v16f32)
14608
0
    return 0;
14609
0
  if ((Subtarget->hasAVX512())) {
14610
0
    return fastEmitInst_rr(X86::VMINPSZrrb, &X86::VR512RegClass, Op0, Op1);
14611
0
  }
14612
0
  return 0;
14613
0
}
14614
14615
0
unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14616
0
  if (RetVT.SimpleTy != MVT::v8f64)
14617
0
    return 0;
14618
0
  if ((Subtarget->hasAVX512())) {
14619
0
    return fastEmitInst_rr(X86::VMINPDZrrb, &X86::VR512RegClass, Op0, Op1);
14620
0
  }
14621
0
  return 0;
14622
0
}
14623
14624
0
unsigned fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14625
0
  switch (VT.SimpleTy) {
14626
0
  case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1);
14627
0
  case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1);
14628
0
  case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1);
14629
0
  default: return 0;
14630
0
  }
14631
0
}
14632
14633
// FastEmit functions for X86ISD::FMULS.
14634
14635
0
unsigned fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14636
0
  if (RetVT.SimpleTy != MVT::v8f16)
14637
0
    return 0;
14638
0
  if ((Subtarget->hasFP16())) {
14639
0
    return fastEmitInst_rr(X86::VMULSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14640
0
  }
14641
0
  return 0;
14642
0
}
14643
14644
0
unsigned fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14645
0
  if (RetVT.SimpleTy != MVT::v4f32)
14646
0
    return 0;
14647
0
  if ((Subtarget->hasAVX512())) {
14648
0
    return fastEmitInst_rr(X86::VMULSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14649
0
  }
14650
0
  return 0;
14651
0
}
14652
14653
0
unsigned fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14654
0
  if (RetVT.SimpleTy != MVT::v2f64)
14655
0
    return 0;
14656
0
  if ((Subtarget->hasAVX512())) {
14657
0
    return fastEmitInst_rr(X86::VMULSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14658
0
  }
14659
0
  return 0;
14660
0
}
14661
14662
0
unsigned fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14663
0
  switch (VT.SimpleTy) {
14664
0
  case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1);
14665
0
  case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1);
14666
0
  case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1);
14667
0
  default: return 0;
14668
0
  }
14669
0
}
14670
14671
// FastEmit functions for X86ISD::FOR.
14672
14673
0
unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14674
0
  if (RetVT.SimpleTy != MVT::f128)
14675
0
    return 0;
14676
0
  if ((Subtarget->hasVLX())) {
14677
0
    return fastEmitInst_rr(X86::VORPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
14678
0
  }
14679
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14680
0
    return fastEmitInst_rr(X86::VORPSrr, &X86::VR128RegClass, Op0, Op1);
14681
0
  }
14682
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14683
0
    return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op1);
14684
0
  }
14685
0
  return 0;
14686
0
}
14687
14688
0
unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14689
0
  if (RetVT.SimpleTy != MVT::v4f32)
14690
0
    return 0;
14691
0
  return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op1);
14692
0
}
14693
14694
0
unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14695
0
  switch (VT.SimpleTy) {
14696
0
  case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1);
14697
0
  case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1);
14698
0
  default: return 0;
14699
0
  }
14700
0
}
14701
14702
// FastEmit functions for X86ISD::FP80_ADD.
14703
14704
0
unsigned fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14705
0
  if (RetVT.SimpleTy != MVT::f80)
14706
0
    return 0;
14707
0
  return fastEmitInst_rr(X86::FP80_ADDr, &X86::RFP80RegClass, Op0, Op1);
14708
0
}
14709
14710
0
unsigned fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14711
0
  switch (VT.SimpleTy) {
14712
0
  case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
14713
0
  default: return 0;
14714
0
  }
14715
0
}
14716
14717
// FastEmit functions for X86ISD::FSQRTS.
14718
14719
0
unsigned fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14720
0
  if (RetVT.SimpleTy != MVT::v8f16)
14721
0
    return 0;
14722
0
  if ((Subtarget->hasFP16())) {
14723
0
    return fastEmitInst_rr(X86::VSQRTSHZr_Int, &X86::VR128XRegClass, Op0, Op1);
14724
0
  }
14725
0
  return 0;
14726
0
}
14727
14728
0
unsigned fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14729
0
  if (RetVT.SimpleTy != MVT::v4f32)
14730
0
    return 0;
14731
0
  if ((Subtarget->hasAVX512())) {
14732
0
    return fastEmitInst_rr(X86::VSQRTSSZr_Int, &X86::VR128XRegClass, Op0, Op1);
14733
0
  }
14734
0
  return 0;
14735
0
}
14736
14737
0
unsigned fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14738
0
  if (RetVT.SimpleTy != MVT::v2f64)
14739
0
    return 0;
14740
0
  if ((Subtarget->hasAVX512())) {
14741
0
    return fastEmitInst_rr(X86::VSQRTSDZr_Int, &X86::VR128XRegClass, Op0, Op1);
14742
0
  }
14743
0
  return 0;
14744
0
}
14745
14746
0
unsigned fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14747
0
  switch (VT.SimpleTy) {
14748
0
  case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1);
14749
0
  case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
14750
0
  case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
14751
0
  default: return 0;
14752
0
  }
14753
0
}
14754
14755
// FastEmit functions for X86ISD::FSUBS.
14756
14757
0
unsigned fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14758
0
  if (RetVT.SimpleTy != MVT::v8f16)
14759
0
    return 0;
14760
0
  if ((Subtarget->hasFP16())) {
14761
0
    return fastEmitInst_rr(X86::VSUBSHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14762
0
  }
14763
0
  return 0;
14764
0
}
14765
14766
0
unsigned fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14767
0
  if (RetVT.SimpleTy != MVT::v4f32)
14768
0
    return 0;
14769
0
  if ((Subtarget->hasAVX512())) {
14770
0
    return fastEmitInst_rr(X86::VSUBSSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14771
0
  }
14772
0
  return 0;
14773
0
}
14774
14775
0
unsigned fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14776
0
  if (RetVT.SimpleTy != MVT::v2f64)
14777
0
    return 0;
14778
0
  if ((Subtarget->hasAVX512())) {
14779
0
    return fastEmitInst_rr(X86::VSUBSDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
14780
0
  }
14781
0
  return 0;
14782
0
}
14783
14784
0
unsigned fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14785
0
  switch (VT.SimpleTy) {
14786
0
  case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1);
14787
0
  case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1);
14788
0
  case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1);
14789
0
  default: return 0;
14790
0
  }
14791
0
}
14792
14793
// FastEmit functions for X86ISD::FXOR.
14794
14795
0
unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14796
0
  if (RetVT.SimpleTy != MVT::f128)
14797
0
    return 0;
14798
0
  if ((Subtarget->hasVLX())) {
14799
0
    return fastEmitInst_rr(X86::VXORPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
14800
0
  }
14801
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14802
0
    return fastEmitInst_rr(X86::VXORPSrr, &X86::VR128RegClass, Op0, Op1);
14803
0
  }
14804
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14805
0
    return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op1);
14806
0
  }
14807
0
  return 0;
14808
0
}
14809
14810
0
unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14811
0
  if (RetVT.SimpleTy != MVT::v4f32)
14812
0
    return 0;
14813
0
  return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op1);
14814
0
}
14815
14816
0
unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14817
0
  switch (VT.SimpleTy) {
14818
0
  case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1);
14819
0
  case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1);
14820
0
  default: return 0;
14821
0
  }
14822
0
}
14823
14824
// FastEmit functions for X86ISD::GF2P8MULB.
14825
14826
0
unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14827
0
  if (RetVT.SimpleTy != MVT::v16i8)
14828
0
    return 0;
14829
0
  if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
14830
0
    return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op1);
14831
0
  }
14832
0
  if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
14833
0
    return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op1);
14834
0
  }
14835
0
  if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14836
0
    return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op1);
14837
0
  }
14838
0
  return 0;
14839
0
}
14840
14841
0
unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14842
0
  if (RetVT.SimpleTy != MVT::v32i8)
14843
0
    return 0;
14844
0
  if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
14845
0
    return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op1);
14846
0
  }
14847
0
  if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) {
14848
0
    return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op1);
14849
0
  }
14850
0
  return 0;
14851
0
}
14852
14853
0
unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14854
0
  if (RetVT.SimpleTy != MVT::v64i8)
14855
0
    return 0;
14856
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) {
14857
0
    return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op1);
14858
0
  }
14859
0
  return 0;
14860
0
}
14861
14862
0
unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14863
0
  switch (VT.SimpleTy) {
14864
0
  case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1);
14865
0
  case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1);
14866
0
  case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1);
14867
0
  default: return 0;
14868
0
  }
14869
0
}
14870
14871
// FastEmit functions for X86ISD::HADD.
14872
14873
0
unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14874
0
  if (RetVT.SimpleTy != MVT::v8i16)
14875
0
    return 0;
14876
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14877
0
    return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op1);
14878
0
  }
14879
0
  if ((Subtarget->hasAVX())) {
14880
0
    return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op1);
14881
0
  }
14882
0
  return 0;
14883
0
}
14884
14885
0
unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14886
0
  if (RetVT.SimpleTy != MVT::v16i16)
14887
0
    return 0;
14888
0
  if ((Subtarget->hasAVX2())) {
14889
0
    return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op1);
14890
0
  }
14891
0
  return 0;
14892
0
}
14893
14894
0
unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14895
0
  if (RetVT.SimpleTy != MVT::v4i32)
14896
0
    return 0;
14897
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14898
0
    return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op1);
14899
0
  }
14900
0
  if ((Subtarget->hasAVX())) {
14901
0
    return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op1);
14902
0
  }
14903
0
  return 0;
14904
0
}
14905
14906
0
unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14907
0
  if (RetVT.SimpleTy != MVT::v8i32)
14908
0
    return 0;
14909
0
  if ((Subtarget->hasAVX2())) {
14910
0
    return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op1);
14911
0
  }
14912
0
  return 0;
14913
0
}
14914
14915
0
unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14916
0
  switch (VT.SimpleTy) {
14917
0
  case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1);
14918
0
  case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1);
14919
0
  case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1);
14920
0
  case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1);
14921
0
  default: return 0;
14922
0
  }
14923
0
}
14924
14925
// FastEmit functions for X86ISD::HSUB.
14926
14927
0
unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14928
0
  if (RetVT.SimpleTy != MVT::v8i16)
14929
0
    return 0;
14930
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14931
0
    return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op1);
14932
0
  }
14933
0
  if ((Subtarget->hasAVX())) {
14934
0
    return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op1);
14935
0
  }
14936
0
  return 0;
14937
0
}
14938
14939
0
unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14940
0
  if (RetVT.SimpleTy != MVT::v16i16)
14941
0
    return 0;
14942
0
  if ((Subtarget->hasAVX2())) {
14943
0
    return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op1);
14944
0
  }
14945
0
  return 0;
14946
0
}
14947
14948
0
unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14949
0
  if (RetVT.SimpleTy != MVT::v4i32)
14950
0
    return 0;
14951
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14952
0
    return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op1);
14953
0
  }
14954
0
  if ((Subtarget->hasAVX())) {
14955
0
    return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op1);
14956
0
  }
14957
0
  return 0;
14958
0
}
14959
14960
0
unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14961
0
  if (RetVT.SimpleTy != MVT::v8i32)
14962
0
    return 0;
14963
0
  if ((Subtarget->hasAVX2())) {
14964
0
    return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op1);
14965
0
  }
14966
0
  return 0;
14967
0
}
14968
14969
0
unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14970
0
  switch (VT.SimpleTy) {
14971
0
  case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1);
14972
0
  case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1);
14973
0
  case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1);
14974
0
  case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1);
14975
0
  default: return 0;
14976
0
  }
14977
0
}
14978
14979
// FastEmit functions for X86ISD::KADD.
14980
14981
0
unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14982
0
  if (RetVT.SimpleTy != MVT::v8i1)
14983
0
    return 0;
14984
0
  if ((Subtarget->hasDQI())) {
14985
0
    return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op1);
14986
0
  }
14987
0
  return 0;
14988
0
}
14989
14990
0
unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14991
0
  if (RetVT.SimpleTy != MVT::v16i1)
14992
0
    return 0;
14993
0
  if ((Subtarget->hasDQI())) {
14994
0
    return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op1);
14995
0
  }
14996
0
  return 0;
14997
0
}
14998
14999
0
unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15000
0
  if (RetVT.SimpleTy != MVT::v32i1)
15001
0
    return 0;
15002
0
  if ((Subtarget->hasBWI())) {
15003
0
    return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op1);
15004
0
  }
15005
0
  return 0;
15006
0
}
15007
15008
0
unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15009
0
  if (RetVT.SimpleTy != MVT::v64i1)
15010
0
    return 0;
15011
0
  if ((Subtarget->hasBWI())) {
15012
0
    return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op1);
15013
0
  }
15014
0
  return 0;
15015
0
}
15016
15017
0
unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15018
0
  switch (VT.SimpleTy) {
15019
0
  case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1);
15020
0
  case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1);
15021
0
  case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1);
15022
0
  case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1);
15023
0
  default: return 0;
15024
0
  }
15025
0
}
15026
15027
// FastEmit functions for X86ISD::KORTEST.
15028
15029
0
unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15030
0
  if (RetVT.SimpleTy != MVT::i32)
15031
0
    return 0;
15032
0
  if ((Subtarget->hasDQI())) {
15033
0
    return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op1);
15034
0
  }
15035
0
  return 0;
15036
0
}
15037
15038
0
unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15039
0
  if (RetVT.SimpleTy != MVT::i32)
15040
0
    return 0;
15041
0
  if ((Subtarget->hasAVX512())) {
15042
0
    return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op1);
15043
0
  }
15044
0
  return 0;
15045
0
}
15046
15047
0
unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15048
0
  if (RetVT.SimpleTy != MVT::i32)
15049
0
    return 0;
15050
0
  if ((Subtarget->hasBWI())) {
15051
0
    return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op1);
15052
0
  }
15053
0
  return 0;
15054
0
}
15055
15056
0
unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15057
0
  if (RetVT.SimpleTy != MVT::i32)
15058
0
    return 0;
15059
0
  if ((Subtarget->hasBWI())) {
15060
0
    return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op1);
15061
0
  }
15062
0
  return 0;
15063
0
}
15064
15065
0
unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15066
0
  switch (VT.SimpleTy) {
15067
0
  case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
15068
0
  case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
15069
0
  case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
15070
0
  case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
15071
0
  default: return 0;
15072
0
  }
15073
0
}
15074
15075
// FastEmit functions for X86ISD::KTEST.
15076
15077
0
unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15078
0
  if (RetVT.SimpleTy != MVT::i32)
15079
0
    return 0;
15080
0
  if ((Subtarget->hasDQI())) {
15081
0
    return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op1);
15082
0
  }
15083
0
  return 0;
15084
0
}
15085
15086
0
unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15087
0
  if (RetVT.SimpleTy != MVT::i32)
15088
0
    return 0;
15089
0
  if ((Subtarget->hasDQI())) {
15090
0
    return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op1);
15091
0
  }
15092
0
  return 0;
15093
0
}
15094
15095
0
unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15096
0
  if (RetVT.SimpleTy != MVT::i32)
15097
0
    return 0;
15098
0
  if ((Subtarget->hasBWI())) {
15099
0
    return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op1);
15100
0
  }
15101
0
  return 0;
15102
0
}
15103
15104
0
unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15105
0
  if (RetVT.SimpleTy != MVT::i32)
15106
0
    return 0;
15107
0
  if ((Subtarget->hasBWI())) {
15108
0
    return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op1);
15109
0
  }
15110
0
  return 0;
15111
0
}
15112
15113
0
unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15114
0
  switch (VT.SimpleTy) {
15115
0
  case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1);
15116
0
  case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1);
15117
0
  case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1);
15118
0
  case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1);
15119
0
  default: return 0;
15120
0
  }
15121
0
}
15122
15123
// FastEmit functions for X86ISD::MOVHLPS.
15124
15125
0
unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15126
0
  if (RetVT.SimpleTy != MVT::v4f32)
15127
0
    return 0;
15128
0
  if ((Subtarget->hasAVX512())) {
15129
0
    return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op1);
15130
0
  }
15131
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15132
0
    return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op1);
15133
0
  }
15134
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15135
0
    return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op1);
15136
0
  }
15137
0
  return 0;
15138
0
}
15139
15140
0
unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15141
0
  switch (VT.SimpleTy) {
15142
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15143
0
  default: return 0;
15144
0
  }
15145
0
}
15146
15147
// FastEmit functions for X86ISD::MOVLHPS.
15148
15149
0
unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15150
0
  if (RetVT.SimpleTy != MVT::v4f32)
15151
0
    return 0;
15152
0
  if ((Subtarget->hasAVX512())) {
15153
0
    return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op1);
15154
0
  }
15155
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15156
0
    return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op1);
15157
0
  }
15158
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15159
0
    return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op1);
15160
0
  }
15161
0
  return 0;
15162
0
}
15163
15164
0
unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15165
0
  switch (VT.SimpleTy) {
15166
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1);
15167
0
  default: return 0;
15168
0
  }
15169
0
}
15170
15171
// FastEmit functions for X86ISD::MOVSD.
15172
15173
0
unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15174
0
  if (RetVT.SimpleTy != MVT::v2f64)
15175
0
    return 0;
15176
0
  if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
15177
0
    return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op1);
15178
0
  }
15179
0
  if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15180
0
    return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op1);
15181
0
  }
15182
0
  if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15183
0
    return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op1);
15184
0
  }
15185
0
  return 0;
15186
0
}
15187
15188
0
unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15189
0
  switch (VT.SimpleTy) {
15190
0
  case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1);
15191
0
  default: return 0;
15192
0
  }
15193
0
}
15194
15195
// FastEmit functions for X86ISD::MOVSH.
15196
15197
0
unsigned fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15198
0
  if (RetVT.SimpleTy != MVT::v8f16)
15199
0
    return 0;
15200
0
  if ((Subtarget->hasFP16())) {
15201
0
    return fastEmitInst_rr(X86::VMOVSHZrr, &X86::VR128XRegClass, Op0, Op1);
15202
0
  }
15203
0
  return 0;
15204
0
}
15205
15206
0
unsigned fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15207
0
  switch (VT.SimpleTy) {
15208
0
  case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1);
15209
0
  default: return 0;
15210
0
  }
15211
0
}
15212
15213
// FastEmit functions for X86ISD::MOVSS.
15214
15215
0
unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15216
0
  if (RetVT.SimpleTy != MVT::v4f32)
15217
0
    return 0;
15218
0
  if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
15219
0
    return fastEmitInst_rr(X86::VMOVSSZrr, &X86::VR128XRegClass, Op0, Op1);
15220
0
  }
15221
0
  if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
15222
0
    return fastEmitInst_rr(X86::MOVSSrr, &X86::VR128RegClass, Op0, Op1);
15223
0
  }
15224
0
  if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
15225
0
    return fastEmitInst_rr(X86::VMOVSSrr, &X86::VR128RegClass, Op0, Op1);
15226
0
  }
15227
0
  return 0;
15228
0
}
15229
15230
0
unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15231
0
  switch (VT.SimpleTy) {
15232
0
  case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1);
15233
0
  default: return 0;
15234
0
  }
15235
0
}
15236
15237
// FastEmit functions for X86ISD::MULHRS.
15238
15239
0
unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15240
0
  if (RetVT.SimpleTy != MVT::v8i16)
15241
0
    return 0;
15242
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15243
0
    return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
15244
0
  }
15245
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
15246
0
    return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op1);
15247
0
  }
15248
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15249
0
    return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op1);
15250
0
  }
15251
0
  return 0;
15252
0
}
15253
15254
0
unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15255
0
  if (RetVT.SimpleTy != MVT::v16i16)
15256
0
    return 0;
15257
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15258
0
    return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
15259
0
  }
15260
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15261
0
    return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op1);
15262
0
  }
15263
0
  return 0;
15264
0
}
15265
15266
0
unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15267
0
  if (RetVT.SimpleTy != MVT::v32i16)
15268
0
    return 0;
15269
0
  if ((Subtarget->hasBWI())) {
15270
0
    return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op1);
15271
0
  }
15272
0
  return 0;
15273
0
}
15274
15275
0
unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15276
0
  switch (VT.SimpleTy) {
15277
0
  case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1);
15278
0
  case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1);
15279
0
  case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1);
15280
0
  default: return 0;
15281
0
  }
15282
0
}
15283
15284
// FastEmit functions for X86ISD::MULTISHIFT.
15285
15286
0
unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15287
0
  if (RetVT.SimpleTy != MVT::v16i8)
15288
0
    return 0;
15289
0
  if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
15290
0
    return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op1);
15291
0
  }
15292
0
  return 0;
15293
0
}
15294
15295
0
unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15296
0
  if (RetVT.SimpleTy != MVT::v32i8)
15297
0
    return 0;
15298
0
  if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
15299
0
    return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op1);
15300
0
  }
15301
0
  return 0;
15302
0
}
15303
15304
0
unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15305
0
  if (RetVT.SimpleTy != MVT::v64i8)
15306
0
    return 0;
15307
0
  if ((Subtarget->hasVBMI())) {
15308
0
    return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op1);
15309
0
  }
15310
0
  return 0;
15311
0
}
15312
15313
0
unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15314
0
  switch (VT.SimpleTy) {
15315
0
  case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1);
15316
0
  case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1);
15317
0
  case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1);
15318
0
  default: return 0;
15319
0
  }
15320
0
}
15321
15322
// FastEmit functions for X86ISD::PACKSS.
15323
15324
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15325
0
  if (RetVT.SimpleTy != MVT::v16i8)
15326
0
    return 0;
15327
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15328
0
    return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op1);
15329
0
  }
15330
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15331
0
    return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op1);
15332
0
  }
15333
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15334
0
    return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op1);
15335
0
  }
15336
0
  return 0;
15337
0
}
15338
15339
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15340
0
  if (RetVT.SimpleTy != MVT::v32i8)
15341
0
    return 0;
15342
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15343
0
    return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op1);
15344
0
  }
15345
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15346
0
    return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op1);
15347
0
  }
15348
0
  return 0;
15349
0
}
15350
15351
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15352
0
  if (RetVT.SimpleTy != MVT::v64i8)
15353
0
    return 0;
15354
0
  if ((Subtarget->hasBWI())) {
15355
0
    return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op1);
15356
0
  }
15357
0
  return 0;
15358
0
}
15359
15360
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15361
0
  if (RetVT.SimpleTy != MVT::v8i16)
15362
0
    return 0;
15363
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15364
0
    return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op1);
15365
0
  }
15366
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15367
0
    return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op1);
15368
0
  }
15369
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15370
0
    return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op1);
15371
0
  }
15372
0
  return 0;
15373
0
}
15374
15375
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15376
0
  if (RetVT.SimpleTy != MVT::v16i16)
15377
0
    return 0;
15378
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15379
0
    return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op1);
15380
0
  }
15381
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15382
0
    return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op1);
15383
0
  }
15384
0
  return 0;
15385
0
}
15386
15387
0
unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15388
0
  if (RetVT.SimpleTy != MVT::v32i16)
15389
0
    return 0;
15390
0
  if ((Subtarget->hasBWI())) {
15391
0
    return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op1);
15392
0
  }
15393
0
  return 0;
15394
0
}
15395
15396
0
unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15397
0
  switch (VT.SimpleTy) {
15398
0
  case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1);
15399
0
  case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1);
15400
0
  case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1);
15401
0
  case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1);
15402
0
  case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1);
15403
0
  case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1);
15404
0
  default: return 0;
15405
0
  }
15406
0
}
15407
15408
// FastEmit functions for X86ISD::PACKUS.
15409
15410
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15411
0
  if (RetVT.SimpleTy != MVT::v16i8)
15412
0
    return 0;
15413
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15414
0
    return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op1);
15415
0
  }
15416
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15417
0
    return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op1);
15418
0
  }
15419
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15420
0
    return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op1);
15421
0
  }
15422
0
  return 0;
15423
0
}
15424
15425
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15426
0
  if (RetVT.SimpleTy != MVT::v32i8)
15427
0
    return 0;
15428
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15429
0
    return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op1);
15430
0
  }
15431
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15432
0
    return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op1);
15433
0
  }
15434
0
  return 0;
15435
0
}
15436
15437
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15438
0
  if (RetVT.SimpleTy != MVT::v64i8)
15439
0
    return 0;
15440
0
  if ((Subtarget->hasBWI())) {
15441
0
    return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op1);
15442
0
  }
15443
0
  return 0;
15444
0
}
15445
15446
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15447
0
  if (RetVT.SimpleTy != MVT::v8i16)
15448
0
    return 0;
15449
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15450
0
    return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op1);
15451
0
  }
15452
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15453
0
    return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op1);
15454
0
  }
15455
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15456
0
    return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op1);
15457
0
  }
15458
0
  return 0;
15459
0
}
15460
15461
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15462
0
  if (RetVT.SimpleTy != MVT::v16i16)
15463
0
    return 0;
15464
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15465
0
    return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op1);
15466
0
  }
15467
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15468
0
    return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op1);
15469
0
  }
15470
0
  return 0;
15471
0
}
15472
15473
0
unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15474
0
  if (RetVT.SimpleTy != MVT::v32i16)
15475
0
    return 0;
15476
0
  if ((Subtarget->hasBWI())) {
15477
0
    return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op1);
15478
0
  }
15479
0
  return 0;
15480
0
}
15481
15482
0
unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15483
0
  switch (VT.SimpleTy) {
15484
0
  case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1);
15485
0
  case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1);
15486
0
  case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1);
15487
0
  case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1);
15488
0
  case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1);
15489
0
  case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1);
15490
0
  default: return 0;
15491
0
  }
15492
0
}
15493
15494
// FastEmit functions for X86ISD::PCMPEQ.
15495
15496
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15497
0
  if (RetVT.SimpleTy != MVT::v16i8)
15498
0
    return 0;
15499
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15500
0
    return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op1);
15501
0
  }
15502
0
  if ((Subtarget->hasAVX()) && (true)) {
15503
0
    return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op1);
15504
0
  }
15505
0
  return 0;
15506
0
}
15507
15508
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15509
0
  if (RetVT.SimpleTy != MVT::v32i8)
15510
0
    return 0;
15511
0
  if ((Subtarget->hasAVX2()) && (true)) {
15512
0
    return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op1);
15513
0
  }
15514
0
  return 0;
15515
0
}
15516
15517
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15518
0
  if (RetVT.SimpleTy != MVT::v8i16)
15519
0
    return 0;
15520
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15521
0
    return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op1);
15522
0
  }
15523
0
  if ((Subtarget->hasAVX()) && (true)) {
15524
0
    return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128RegClass, Op0, Op1);
15525
0
  }
15526
0
  return 0;
15527
0
}
15528
15529
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15530
0
  if (RetVT.SimpleTy != MVT::v16i16)
15531
0
    return 0;
15532
0
  if ((Subtarget->hasAVX2()) && (true)) {
15533
0
    return fastEmitInst_rr(X86::VPCMPEQWYrr, &X86::VR256RegClass, Op0, Op1);
15534
0
  }
15535
0
  return 0;
15536
0
}
15537
15538
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15539
0
  if (RetVT.SimpleTy != MVT::v4i32)
15540
0
    return 0;
15541
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15542
0
    return fastEmitInst_rr(X86::PCMPEQDrr, &X86::VR128RegClass, Op0, Op1);
15543
0
  }
15544
0
  if ((Subtarget->hasAVX()) && (true)) {
15545
0
    return fastEmitInst_rr(X86::VPCMPEQDrr, &X86::VR128RegClass, Op0, Op1);
15546
0
  }
15547
0
  return 0;
15548
0
}
15549
15550
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15551
0
  if (RetVT.SimpleTy != MVT::v8i32)
15552
0
    return 0;
15553
0
  if ((Subtarget->hasAVX2()) && (true)) {
15554
0
    return fastEmitInst_rr(X86::VPCMPEQDYrr, &X86::VR256RegClass, Op0, Op1);
15555
0
  }
15556
0
  return 0;
15557
0
}
15558
15559
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15560
0
  if (RetVT.SimpleTy != MVT::v2i64)
15561
0
    return 0;
15562
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15563
0
    return fastEmitInst_rr(X86::PCMPEQQrr, &X86::VR128RegClass, Op0, Op1);
15564
0
  }
15565
0
  if ((Subtarget->hasAVX())) {
15566
0
    return fastEmitInst_rr(X86::VPCMPEQQrr, &X86::VR128RegClass, Op0, Op1);
15567
0
  }
15568
0
  return 0;
15569
0
}
15570
15571
0
unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15572
0
  if (RetVT.SimpleTy != MVT::v4i64)
15573
0
    return 0;
15574
0
  if ((Subtarget->hasAVX2())) {
15575
0
    return fastEmitInst_rr(X86::VPCMPEQQYrr, &X86::VR256RegClass, Op0, Op1);
15576
0
  }
15577
0
  return 0;
15578
0
}
15579
15580
0
unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15581
0
  switch (VT.SimpleTy) {
15582
0
  case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
15583
0
  case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1);
15584
0
  case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
15585
0
  case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1);
15586
0
  case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
15587
0
  case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1);
15588
0
  case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15589
0
  case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15590
0
  default: return 0;
15591
0
  }
15592
0
}
15593
15594
// FastEmit functions for X86ISD::PCMPGT.
15595
15596
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15597
0
  if (RetVT.SimpleTy != MVT::v16i8)
15598
0
    return 0;
15599
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15600
0
    return fastEmitInst_rr(X86::PCMPGTBrr, &X86::VR128RegClass, Op0, Op1);
15601
0
  }
15602
0
  if ((Subtarget->hasAVX()) && (true)) {
15603
0
    return fastEmitInst_rr(X86::VPCMPGTBrr, &X86::VR128RegClass, Op0, Op1);
15604
0
  }
15605
0
  return 0;
15606
0
}
15607
15608
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15609
0
  if (RetVT.SimpleTy != MVT::v32i8)
15610
0
    return 0;
15611
0
  if ((Subtarget->hasAVX2()) && (true)) {
15612
0
    return fastEmitInst_rr(X86::VPCMPGTBYrr, &X86::VR256RegClass, Op0, Op1);
15613
0
  }
15614
0
  return 0;
15615
0
}
15616
15617
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15618
0
  if (RetVT.SimpleTy != MVT::v8i16)
15619
0
    return 0;
15620
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15621
0
    return fastEmitInst_rr(X86::PCMPGTWrr, &X86::VR128RegClass, Op0, Op1);
15622
0
  }
15623
0
  if ((Subtarget->hasAVX()) && (true)) {
15624
0
    return fastEmitInst_rr(X86::VPCMPGTWrr, &X86::VR128RegClass, Op0, Op1);
15625
0
  }
15626
0
  return 0;
15627
0
}
15628
15629
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15630
0
  if (RetVT.SimpleTy != MVT::v16i16)
15631
0
    return 0;
15632
0
  if ((Subtarget->hasAVX2()) && (true)) {
15633
0
    return fastEmitInst_rr(X86::VPCMPGTWYrr, &X86::VR256RegClass, Op0, Op1);
15634
0
  }
15635
0
  return 0;
15636
0
}
15637
15638
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15639
0
  if (RetVT.SimpleTy != MVT::v4i32)
15640
0
    return 0;
15641
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15642
0
    return fastEmitInst_rr(X86::PCMPGTDrr, &X86::VR128RegClass, Op0, Op1);
15643
0
  }
15644
0
  if ((Subtarget->hasAVX()) && (true)) {
15645
0
    return fastEmitInst_rr(X86::VPCMPGTDrr, &X86::VR128RegClass, Op0, Op1);
15646
0
  }
15647
0
  return 0;
15648
0
}
15649
15650
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15651
0
  if (RetVT.SimpleTy != MVT::v8i32)
15652
0
    return 0;
15653
0
  if ((Subtarget->hasAVX2()) && (true)) {
15654
0
    return fastEmitInst_rr(X86::VPCMPGTDYrr, &X86::VR256RegClass, Op0, Op1);
15655
0
  }
15656
0
  return 0;
15657
0
}
15658
15659
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15660
0
  if (RetVT.SimpleTy != MVT::v2i64)
15661
0
    return 0;
15662
0
  if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
15663
0
    return fastEmitInst_rr(X86::PCMPGTQrr, &X86::VR128RegClass, Op0, Op1);
15664
0
  }
15665
0
  if ((Subtarget->hasAVX())) {
15666
0
    return fastEmitInst_rr(X86::VPCMPGTQrr, &X86::VR128RegClass, Op0, Op1);
15667
0
  }
15668
0
  return 0;
15669
0
}
15670
15671
0
unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15672
0
  if (RetVT.SimpleTy != MVT::v4i64)
15673
0
    return 0;
15674
0
  if ((Subtarget->hasAVX2())) {
15675
0
    return fastEmitInst_rr(X86::VPCMPGTQYrr, &X86::VR256RegClass, Op0, Op1);
15676
0
  }
15677
0
  return 0;
15678
0
}
15679
15680
0
unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15681
0
  switch (VT.SimpleTy) {
15682
0
  case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1);
15683
0
  case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1);
15684
0
  case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1);
15685
0
  case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1);
15686
0
  case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1);
15687
0
  case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1);
15688
0
  case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1);
15689
0
  case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1);
15690
0
  default: return 0;
15691
0
  }
15692
0
}
15693
15694
// FastEmit functions for X86ISD::PDEP.
15695
15696
0
unsigned fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15697
0
  if (RetVT.SimpleTy != MVT::i32)
15698
0
    return 0;
15699
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15700
0
    return fastEmitInst_rr(X86::PDEP32rr_EVEX, &X86::GR32RegClass, Op0, Op1);
15701
0
  }
15702
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15703
0
    return fastEmitInst_rr(X86::PDEP32rr, &X86::GR32RegClass, Op0, Op1);
15704
0
  }
15705
0
  return 0;
15706
0
}
15707
15708
0
unsigned fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15709
0
  if (RetVT.SimpleTy != MVT::i64)
15710
0
    return 0;
15711
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15712
0
    return fastEmitInst_rr(X86::PDEP64rr_EVEX, &X86::GR64RegClass, Op0, Op1);
15713
0
  }
15714
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15715
0
    return fastEmitInst_rr(X86::PDEP64rr, &X86::GR64RegClass, Op0, Op1);
15716
0
  }
15717
0
  return 0;
15718
0
}
15719
15720
0
unsigned fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15721
0
  switch (VT.SimpleTy) {
15722
0
  case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1);
15723
0
  case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1);
15724
0
  default: return 0;
15725
0
  }
15726
0
}
15727
15728
// FastEmit functions for X86ISD::PEXT.
15729
15730
0
unsigned fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15731
0
  if (RetVT.SimpleTy != MVT::i32)
15732
0
    return 0;
15733
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15734
0
    return fastEmitInst_rr(X86::PEXT32rr_EVEX, &X86::GR32RegClass, Op0, Op1);
15735
0
  }
15736
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15737
0
    return fastEmitInst_rr(X86::PEXT32rr, &X86::GR32RegClass, Op0, Op1);
15738
0
  }
15739
0
  return 0;
15740
0
}
15741
15742
0
unsigned fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15743
0
  if (RetVT.SimpleTy != MVT::i64)
15744
0
    return 0;
15745
0
  if ((Subtarget->hasBMI2()) && (Subtarget->hasEGPR())) {
15746
0
    return fastEmitInst_rr(X86::PEXT64rr_EVEX, &X86::GR64RegClass, Op0, Op1);
15747
0
  }
15748
0
  if ((Subtarget->hasBMI2()) && (!Subtarget->hasEGPR())) {
15749
0
    return fastEmitInst_rr(X86::PEXT64rr, &X86::GR64RegClass, Op0, Op1);
15750
0
  }
15751
0
  return 0;
15752
0
}
15753
15754
0
unsigned fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15755
0
  switch (VT.SimpleTy) {
15756
0
  case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1);
15757
0
  case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1);
15758
0
  default: return 0;
15759
0
  }
15760
0
}
15761
15762
// FastEmit functions for X86ISD::PMULDQ.
15763
15764
0
unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15765
0
  if (RetVT.SimpleTy != MVT::v2i64)
15766
0
    return 0;
15767
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15768
0
    return fastEmitInst_rr(X86::VPMULDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
15769
0
  }
15770
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15771
0
    return fastEmitInst_rr(X86::PMULDQrr, &X86::VR128RegClass, Op0, Op1);
15772
0
  }
15773
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15774
0
    return fastEmitInst_rr(X86::VPMULDQrr, &X86::VR128RegClass, Op0, Op1);
15775
0
  }
15776
0
  return 0;
15777
0
}
15778
15779
0
unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15780
0
  if (RetVT.SimpleTy != MVT::v4i64)
15781
0
    return 0;
15782
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15783
0
    return fastEmitInst_rr(X86::VPMULDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
15784
0
  }
15785
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15786
0
    return fastEmitInst_rr(X86::VPMULDQYrr, &X86::VR256RegClass, Op0, Op1);
15787
0
  }
15788
0
  return 0;
15789
0
}
15790
15791
0
unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15792
0
  if (RetVT.SimpleTy != MVT::v8i64)
15793
0
    return 0;
15794
0
  if ((Subtarget->hasAVX512())) {
15795
0
    return fastEmitInst_rr(X86::VPMULDQZrr, &X86::VR512RegClass, Op0, Op1);
15796
0
  }
15797
0
  return 0;
15798
0
}
15799
15800
0
unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15801
0
  switch (VT.SimpleTy) {
15802
0
  case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15803
0
  case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15804
0
  case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
15805
0
  default: return 0;
15806
0
  }
15807
0
}
15808
15809
// FastEmit functions for X86ISD::PMULUDQ.
15810
15811
0
unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15812
0
  if (RetVT.SimpleTy != MVT::v2i64)
15813
0
    return 0;
15814
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15815
0
    return fastEmitInst_rr(X86::VPMULUDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
15816
0
  }
15817
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15818
0
    return fastEmitInst_rr(X86::PMULUDQrr, &X86::VR128RegClass, Op0, Op1);
15819
0
  }
15820
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15821
0
    return fastEmitInst_rr(X86::VPMULUDQrr, &X86::VR128RegClass, Op0, Op1);
15822
0
  }
15823
0
  return 0;
15824
0
}
15825
15826
0
unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15827
0
  if (RetVT.SimpleTy != MVT::v4i64)
15828
0
    return 0;
15829
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15830
0
    return fastEmitInst_rr(X86::VPMULUDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
15831
0
  }
15832
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15833
0
    return fastEmitInst_rr(X86::VPMULUDQYrr, &X86::VR256RegClass, Op0, Op1);
15834
0
  }
15835
0
  return 0;
15836
0
}
15837
15838
0
unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15839
0
  if (RetVT.SimpleTy != MVT::v8i64)
15840
0
    return 0;
15841
0
  if ((Subtarget->hasAVX512())) {
15842
0
    return fastEmitInst_rr(X86::VPMULUDQZrr, &X86::VR512RegClass, Op0, Op1);
15843
0
  }
15844
0
  return 0;
15845
0
}
15846
15847
0
unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15848
0
  switch (VT.SimpleTy) {
15849
0
  case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1);
15850
0
  case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1);
15851
0
  case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1);
15852
0
  default: return 0;
15853
0
  }
15854
0
}
15855
15856
// FastEmit functions for X86ISD::PSADBW.
15857
15858
0
unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15859
0
  if (RetVT.SimpleTy != MVT::v2i64)
15860
0
    return 0;
15861
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15862
0
    return fastEmitInst_rr(X86::VPSADBWZ128rr, &X86::VR128XRegClass, Op0, Op1);
15863
0
  }
15864
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15865
0
    return fastEmitInst_rr(X86::PSADBWrr, &X86::VR128RegClass, Op0, Op1);
15866
0
  }
15867
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15868
0
    return fastEmitInst_rr(X86::VPSADBWrr, &X86::VR128RegClass, Op0, Op1);
15869
0
  }
15870
0
  return 0;
15871
0
}
15872
15873
0
unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15874
0
  if (RetVT.SimpleTy != MVT::v4i64)
15875
0
    return 0;
15876
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15877
0
    return fastEmitInst_rr(X86::VPSADBWZ256rr, &X86::VR256XRegClass, Op0, Op1);
15878
0
  }
15879
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15880
0
    return fastEmitInst_rr(X86::VPSADBWYrr, &X86::VR256RegClass, Op0, Op1);
15881
0
  }
15882
0
  return 0;
15883
0
}
15884
15885
0
unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15886
0
  if (RetVT.SimpleTy != MVT::v8i64)
15887
0
    return 0;
15888
0
  if ((Subtarget->hasBWI())) {
15889
0
    return fastEmitInst_rr(X86::VPSADBWZrr, &X86::VR512RegClass, Op0, Op1);
15890
0
  }
15891
0
  return 0;
15892
0
}
15893
15894
0
unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15895
0
  switch (VT.SimpleTy) {
15896
0
  case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1);
15897
0
  case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1);
15898
0
  case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1);
15899
0
  default: return 0;
15900
0
  }
15901
0
}
15902
15903
// FastEmit functions for X86ISD::PSHUFB.
15904
15905
0
unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15906
0
  if (RetVT.SimpleTy != MVT::v16i8)
15907
0
    return 0;
15908
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15909
0
    return fastEmitInst_rr(X86::VPSHUFBZ128rr, &X86::VR128XRegClass, Op0, Op1);
15910
0
  }
15911
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
15912
0
    return fastEmitInst_rr(X86::PSHUFBrr, &X86::VR128RegClass, Op0, Op1);
15913
0
  }
15914
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15915
0
    return fastEmitInst_rr(X86::VPSHUFBrr, &X86::VR128RegClass, Op0, Op1);
15916
0
  }
15917
0
  return 0;
15918
0
}
15919
15920
0
unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15921
0
  if (RetVT.SimpleTy != MVT::v32i8)
15922
0
    return 0;
15923
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15924
0
    return fastEmitInst_rr(X86::VPSHUFBZ256rr, &X86::VR256XRegClass, Op0, Op1);
15925
0
  }
15926
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15927
0
    return fastEmitInst_rr(X86::VPSHUFBYrr, &X86::VR256RegClass, Op0, Op1);
15928
0
  }
15929
0
  return 0;
15930
0
}
15931
15932
0
unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15933
0
  if (RetVT.SimpleTy != MVT::v64i8)
15934
0
    return 0;
15935
0
  if ((Subtarget->hasBWI())) {
15936
0
    return fastEmitInst_rr(X86::VPSHUFBZrr, &X86::VR512RegClass, Op0, Op1);
15937
0
  }
15938
0
  return 0;
15939
0
}
15940
15941
0
unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15942
0
  switch (VT.SimpleTy) {
15943
0
  case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1);
15944
0
  case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1);
15945
0
  case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1);
15946
0
  default: return 0;
15947
0
  }
15948
0
}
15949
15950
// FastEmit functions for X86ISD::PTEST.
15951
15952
0
unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15953
0
  if (RetVT.SimpleTy != MVT::i32)
15954
0
    return 0;
15955
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15956
0
    return fastEmitInst_rr(X86::PTESTrr, &X86::VR128RegClass, Op0, Op1);
15957
0
  }
15958
0
  if ((Subtarget->hasAVX())) {
15959
0
    return fastEmitInst_rr(X86::VPTESTrr, &X86::VR128RegClass, Op0, Op1);
15960
0
  }
15961
0
  return 0;
15962
0
}
15963
15964
0
unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15965
0
  if (RetVT.SimpleTy != MVT::i32)
15966
0
    return 0;
15967
0
  if ((Subtarget->hasAVX())) {
15968
0
    return fastEmitInst_rr(X86::VPTESTYrr, &X86::VR256RegClass, Op0, Op1);
15969
0
  }
15970
0
  return 0;
15971
0
}
15972
15973
0
unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
15974
0
  switch (VT.SimpleTy) {
15975
0
  case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1);
15976
0
  case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1);
15977
0
  default: return 0;
15978
0
  }
15979
0
}
15980
15981
// FastEmit functions for X86ISD::RCP14S.
15982
15983
0
unsigned fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15984
0
  if (RetVT.SimpleTy != MVT::v8f16)
15985
0
    return 0;
15986
0
  if ((Subtarget->hasFP16())) {
15987
0
    return fastEmitInst_rr(X86::VRCPSHZrr, &X86::VR128XRegClass, Op0, Op1);
15988
0
  }
15989
0
  return 0;
15990
0
}
15991
15992
0
unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
15993
0
  if (RetVT.SimpleTy != MVT::v4f32)
15994
0
    return 0;
15995
0
  if ((Subtarget->hasAVX512())) {
15996
0
    return fastEmitInst_rr(X86::VRCP14SSZrr, &X86::VR128XRegClass, Op0, Op1);
15997
0
  }
15998
0
  return 0;
15999
0
}
16000
16001
0
unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16002
0
  if (RetVT.SimpleTy != MVT::v2f64)
16003
0
    return 0;
16004
0
  if ((Subtarget->hasAVX512())) {
16005
0
    return fastEmitInst_rr(X86::VRCP14SDZrr, &X86::VR128XRegClass, Op0, Op1);
16006
0
  }
16007
0
  return 0;
16008
0
}
16009
16010
0
unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16011
0
  switch (VT.SimpleTy) {
16012
0
  case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1);
16013
0
  case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1);
16014
0
  case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1);
16015
0
  default: return 0;
16016
0
  }
16017
0
}
16018
16019
// FastEmit functions for X86ISD::RCP28S.
16020
16021
0
unsigned fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16022
0
  if (RetVT.SimpleTy != MVT::v4f32)
16023
0
    return 0;
16024
0
  if ((Subtarget->hasERI())) {
16025
0
    return fastEmitInst_rr(X86::VRCP28SSZr, &X86::VR128XRegClass, Op0, Op1);
16026
0
  }
16027
0
  return 0;
16028
0
}
16029
16030
0
unsigned fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16031
0
  if (RetVT.SimpleTy != MVT::v2f64)
16032
0
    return 0;
16033
0
  if ((Subtarget->hasERI())) {
16034
0
    return fastEmitInst_rr(X86::VRCP28SDZr, &X86::VR128XRegClass, Op0, Op1);
16035
0
  }
16036
0
  return 0;
16037
0
}
16038
16039
0
unsigned fastEmit_X86ISD_RCP28S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16040
0
  switch (VT.SimpleTy) {
16041
0
  case MVT::v4f32: return fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(RetVT, Op0, Op1);
16042
0
  case MVT::v2f64: return fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(RetVT, Op0, Op1);
16043
0
  default: return 0;
16044
0
  }
16045
0
}
16046
16047
// FastEmit functions for X86ISD::RCP28S_SAE.
16048
16049
0
unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16050
0
  if (RetVT.SimpleTy != MVT::v4f32)
16051
0
    return 0;
16052
0
  if ((Subtarget->hasERI())) {
16053
0
    return fastEmitInst_rr(X86::VRCP28SSZrb, &X86::VR128XRegClass, Op0, Op1);
16054
0
  }
16055
0
  return 0;
16056
0
}
16057
16058
0
unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16059
0
  if (RetVT.SimpleTy != MVT::v2f64)
16060
0
    return 0;
16061
0
  if ((Subtarget->hasERI())) {
16062
0
    return fastEmitInst_rr(X86::VRCP28SDZrb, &X86::VR128XRegClass, Op0, Op1);
16063
0
  }
16064
0
  return 0;
16065
0
}
16066
16067
0
unsigned fastEmit_X86ISD_RCP28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16068
0
  switch (VT.SimpleTy) {
16069
0
  case MVT::v4f32: return fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16070
0
  case MVT::v2f64: return fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16071
0
  default: return 0;
16072
0
  }
16073
0
}
16074
16075
// FastEmit functions for X86ISD::RSQRT14S.
16076
16077
0
unsigned fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16078
0
  if (RetVT.SimpleTy != MVT::v8f16)
16079
0
    return 0;
16080
0
  if ((Subtarget->hasFP16())) {
16081
0
    return fastEmitInst_rr(X86::VRSQRTSHZrr, &X86::VR128XRegClass, Op0, Op1);
16082
0
  }
16083
0
  return 0;
16084
0
}
16085
16086
0
unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16087
0
  if (RetVT.SimpleTy != MVT::v4f32)
16088
0
    return 0;
16089
0
  if ((Subtarget->hasAVX512())) {
16090
0
    return fastEmitInst_rr(X86::VRSQRT14SSZrr, &X86::VR128XRegClass, Op0, Op1);
16091
0
  }
16092
0
  return 0;
16093
0
}
16094
16095
0
unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16096
0
  if (RetVT.SimpleTy != MVT::v2f64)
16097
0
    return 0;
16098
0
  if ((Subtarget->hasAVX512())) {
16099
0
    return fastEmitInst_rr(X86::VRSQRT14SDZrr, &X86::VR128XRegClass, Op0, Op1);
16100
0
  }
16101
0
  return 0;
16102
0
}
16103
16104
0
unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16105
0
  switch (VT.SimpleTy) {
16106
0
  case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1);
16107
0
  case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1);
16108
0
  case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1);
16109
0
  default: return 0;
16110
0
  }
16111
0
}
16112
16113
// FastEmit functions for X86ISD::RSQRT28S.
16114
16115
0
unsigned fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16116
0
  if (RetVT.SimpleTy != MVT::v4f32)
16117
0
    return 0;
16118
0
  if ((Subtarget->hasERI())) {
16119
0
    return fastEmitInst_rr(X86::VRSQRT28SSZr, &X86::VR128XRegClass, Op0, Op1);
16120
0
  }
16121
0
  return 0;
16122
0
}
16123
16124
0
unsigned fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16125
0
  if (RetVT.SimpleTy != MVT::v2f64)
16126
0
    return 0;
16127
0
  if ((Subtarget->hasERI())) {
16128
0
    return fastEmitInst_rr(X86::VRSQRT28SDZr, &X86::VR128XRegClass, Op0, Op1);
16129
0
  }
16130
0
  return 0;
16131
0
}
16132
16133
0
unsigned fastEmit_X86ISD_RSQRT28S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16134
0
  switch (VT.SimpleTy) {
16135
0
  case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(RetVT, Op0, Op1);
16136
0
  case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(RetVT, Op0, Op1);
16137
0
  default: return 0;
16138
0
  }
16139
0
}
16140
16141
// FastEmit functions for X86ISD::RSQRT28S_SAE.
16142
16143
0
unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16144
0
  if (RetVT.SimpleTy != MVT::v4f32)
16145
0
    return 0;
16146
0
  if ((Subtarget->hasERI())) {
16147
0
    return fastEmitInst_rr(X86::VRSQRT28SSZrb, &X86::VR128XRegClass, Op0, Op1);
16148
0
  }
16149
0
  return 0;
16150
0
}
16151
16152
0
unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16153
0
  if (RetVT.SimpleTy != MVT::v2f64)
16154
0
    return 0;
16155
0
  if ((Subtarget->hasERI())) {
16156
0
    return fastEmitInst_rr(X86::VRSQRT28SDZrb, &X86::VR128XRegClass, Op0, Op1);
16157
0
  }
16158
0
  return 0;
16159
0
}
16160
16161
0
unsigned fastEmit_X86ISD_RSQRT28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16162
0
  switch (VT.SimpleTy) {
16163
0
  case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op1);
16164
0
  case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op1);
16165
0
  default: return 0;
16166
0
  }
16167
0
}
16168
16169
// FastEmit functions for X86ISD::SCALEF.
16170
16171
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16172
0
  if (RetVT.SimpleTy != MVT::v8f16)
16173
0
    return 0;
16174
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16175
0
    return fastEmitInst_rr(X86::VSCALEFPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
16176
0
  }
16177
0
  return 0;
16178
0
}
16179
16180
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16181
0
  if (RetVT.SimpleTy != MVT::v16f16)
16182
0
    return 0;
16183
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
16184
0
    return fastEmitInst_rr(X86::VSCALEFPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
16185
0
  }
16186
0
  return 0;
16187
0
}
16188
16189
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16190
0
  if (RetVT.SimpleTy != MVT::v32f16)
16191
0
    return 0;
16192
0
  if ((Subtarget->hasFP16())) {
16193
0
    return fastEmitInst_rr(X86::VSCALEFPHZrr, &X86::VR512RegClass, Op0, Op1);
16194
0
  }
16195
0
  return 0;
16196
0
}
16197
16198
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16199
0
  if (RetVT.SimpleTy != MVT::v4f32)
16200
0
    return 0;
16201
0
  if ((Subtarget->hasVLX())) {
16202
0
    return fastEmitInst_rr(X86::VSCALEFPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
16203
0
  }
16204
0
  return 0;
16205
0
}
16206
16207
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16208
0
  if (RetVT.SimpleTy != MVT::v8f32)
16209
0
    return 0;
16210
0
  if ((Subtarget->hasVLX())) {
16211
0
    return fastEmitInst_rr(X86::VSCALEFPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
16212
0
  }
16213
0
  return 0;
16214
0
}
16215
16216
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16217
0
  if (RetVT.SimpleTy != MVT::v16f32)
16218
0
    return 0;
16219
0
  if ((Subtarget->hasAVX512())) {
16220
0
    return fastEmitInst_rr(X86::VSCALEFPSZrr, &X86::VR512RegClass, Op0, Op1);
16221
0
  }
16222
0
  return 0;
16223
0
}
16224
16225
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16226
0
  if (RetVT.SimpleTy != MVT::v2f64)
16227
0
    return 0;
16228
0
  if ((Subtarget->hasVLX())) {
16229
0
    return fastEmitInst_rr(X86::VSCALEFPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
16230
0
  }
16231
0
  return 0;
16232
0
}
16233
16234
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16235
0
  if (RetVT.SimpleTy != MVT::v4f64)
16236
0
    return 0;
16237
0
  if ((Subtarget->hasVLX())) {
16238
0
    return fastEmitInst_rr(X86::VSCALEFPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
16239
0
  }
16240
0
  return 0;
16241
0
}
16242
16243
0
unsigned fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16244
0
  if (RetVT.SimpleTy != MVT::v8f64)
16245
0
    return 0;
16246
0
  if ((Subtarget->hasAVX512())) {
16247
0
    return fastEmitInst_rr(X86::VSCALEFPDZrr, &X86::VR512RegClass, Op0, Op1);
16248
0
  }
16249
0
  return 0;
16250
0
}
16251
16252
0
unsigned fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16253
0
  switch (VT.SimpleTy) {
16254
0
  case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1);
16255
0
  case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1);
16256
0
  case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1);
16257
0
  case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1);
16258
0
  case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1);
16259
0
  case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1);
16260
0
  case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1);
16261
0
  case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1);
16262
0
  case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1);
16263
0
  default: return 0;
16264
0
  }
16265
0
}
16266
16267
// FastEmit functions for X86ISD::SCALEFS.
16268
16269
0
unsigned fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16270
0
  if (RetVT.SimpleTy != MVT::v8f16)
16271
0
    return 0;
16272
0
  if ((Subtarget->hasFP16())) {
16273
0
    return fastEmitInst_rr(X86::VSCALEFSHZrr, &X86::VR128XRegClass, Op0, Op1);
16274
0
  }
16275
0
  return 0;
16276
0
}
16277
16278
0
unsigned fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16279
0
  if (RetVT.SimpleTy != MVT::v4f32)
16280
0
    return 0;
16281
0
  if ((Subtarget->hasAVX512())) {
16282
0
    return fastEmitInst_rr(X86::VSCALEFSSZrr, &X86::VR128XRegClass, Op0, Op1);
16283
0
  }
16284
0
  return 0;
16285
0
}
16286
16287
0
unsigned fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16288
0
  if (RetVT.SimpleTy != MVT::v2f64)
16289
0
    return 0;
16290
0
  if ((Subtarget->hasAVX512())) {
16291
0
    return fastEmitInst_rr(X86::VSCALEFSDZrr, &X86::VR128XRegClass, Op0, Op1);
16292
0
  }
16293
0
  return 0;
16294
0
}
16295
16296
0
unsigned fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16297
0
  switch (VT.SimpleTy) {
16298
0
  case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1);
16299
0
  case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1);
16300
0
  case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1);
16301
0
  default: return 0;
16302
0
  }
16303
0
}
16304
16305
// FastEmit functions for X86ISD::STRICT_FCMP.
16306
16307
0
unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16308
0
  if (RetVT.SimpleTy != MVT::i32)
16309
0
    return 0;
16310
0
  if ((Subtarget->hasFP16())) {
16311
0
    return fastEmitInst_rr(X86::VUCOMISHZrr, &X86::FR16XRegClass, Op0, Op1);
16312
0
  }
16313
0
  return 0;
16314
0
}
16315
16316
0
unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16317
0
  if (RetVT.SimpleTy != MVT::i32)
16318
0
    return 0;
16319
0
  if ((Subtarget->hasAVX512())) {
16320
0
    return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op1);
16321
0
  }
16322
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16323
0
    return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op1);
16324
0
  }
16325
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16326
0
    return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op1);
16327
0
  }
16328
0
  if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
16329
0
    return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op1);
16330
0
  }
16331
0
  return 0;
16332
0
}
16333
16334
0
unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16335
0
  if (RetVT.SimpleTy != MVT::i32)
16336
0
    return 0;
16337
0
  if ((Subtarget->hasAVX512())) {
16338
0
    return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op1);
16339
0
  }
16340
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16341
0
    return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op1);
16342
0
  }
16343
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16344
0
    return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op1);
16345
0
  }
16346
0
  if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
16347
0
    return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op1);
16348
0
  }
16349
0
  return 0;
16350
0
}
16351
16352
0
unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16353
0
  if (RetVT.SimpleTy != MVT::i32)
16354
0
    return 0;
16355
0
  if ((Subtarget->canUseCMOV())) {
16356
0
    return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op1);
16357
0
  }
16358
0
  return 0;
16359
0
}
16360
16361
0
unsigned fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16362
0
  switch (VT.SimpleTy) {
16363
0
  case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
16364
0
  case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
16365
0
  case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
16366
0
  case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1);
16367
0
  default: return 0;
16368
0
  }
16369
0
}
16370
16371
// FastEmit functions for X86ISD::STRICT_FCMPS.
16372
16373
0
unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16374
0
  if (RetVT.SimpleTy != MVT::i32)
16375
0
    return 0;
16376
0
  if ((Subtarget->hasFP16())) {
16377
0
    return fastEmitInst_rr(X86::VCOMISHZrr, &X86::FR16XRegClass, Op0, Op1);
16378
0
  }
16379
0
  return 0;
16380
0
}
16381
16382
0
unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16383
0
  if (RetVT.SimpleTy != MVT::i32)
16384
0
    return 0;
16385
0
  if ((Subtarget->hasAVX512())) {
16386
0
    return fastEmitInst_rr(X86::VCOMISSZrr, &X86::FR32XRegClass, Op0, Op1);
16387
0
  }
16388
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16389
0
    return fastEmitInst_rr(X86::COMISSrr, &X86::FR32RegClass, Op0, Op1);
16390
0
  }
16391
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16392
0
    return fastEmitInst_rr(X86::VCOMISSrr, &X86::FR32RegClass, Op0, Op1);
16393
0
  }
16394
0
  if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) {
16395
0
    return fastEmitInst_rr(X86::COM_FpIr32, &X86::RFP32RegClass, Op0, Op1);
16396
0
  }
16397
0
  return 0;
16398
0
}
16399
16400
0
unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16401
0
  if (RetVT.SimpleTy != MVT::i32)
16402
0
    return 0;
16403
0
  if ((Subtarget->hasAVX512())) {
16404
0
    return fastEmitInst_rr(X86::VCOMISDZrr, &X86::FR64XRegClass, Op0, Op1);
16405
0
  }
16406
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16407
0
    return fastEmitInst_rr(X86::COMISDrr, &X86::FR64RegClass, Op0, Op1);
16408
0
  }
16409
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16410
0
    return fastEmitInst_rr(X86::VCOMISDrr, &X86::FR64RegClass, Op0, Op1);
16411
0
  }
16412
0
  if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) {
16413
0
    return fastEmitInst_rr(X86::COM_FpIr64, &X86::RFP64RegClass, Op0, Op1);
16414
0
  }
16415
0
  return 0;
16416
0
}
16417
16418
0
unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16419
0
  if (RetVT.SimpleTy != MVT::i32)
16420
0
    return 0;
16421
0
  if ((Subtarget->canUseCMOV())) {
16422
0
    return fastEmitInst_rr(X86::COM_FpIr80, &X86::RFP80RegClass, Op0, Op1);
16423
0
  }
16424
0
  return 0;
16425
0
}
16426
16427
0
unsigned fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16428
0
  switch (VT.SimpleTy) {
16429
0
  case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1);
16430
0
  case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1);
16431
0
  case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1);
16432
0
  case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1);
16433
0
  default: return 0;
16434
0
  }
16435
0
}
16436
16437
// FastEmit functions for X86ISD::STRICT_FP80_ADD.
16438
16439
0
unsigned fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16440
0
  if (RetVT.SimpleTy != MVT::f80)
16441
0
    return 0;
16442
0
  return fastEmitInst_rr(X86::FP80_ADDr, &X86::RFP80RegClass, Op0, Op1);
16443
0
}
16444
16445
0
unsigned fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16446
0
  switch (VT.SimpleTy) {
16447
0
  case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1);
16448
0
  default: return 0;
16449
0
  }
16450
0
}
16451
16452
// FastEmit functions for X86ISD::TESTP.
16453
16454
0
unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16455
0
  if (RetVT.SimpleTy != MVT::i32)
16456
0
    return 0;
16457
0
  if ((Subtarget->hasAVX())) {
16458
0
    return fastEmitInst_rr(X86::VTESTPSrr, &X86::VR128RegClass, Op0, Op1);
16459
0
  }
16460
0
  return 0;
16461
0
}
16462
16463
0
unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16464
0
  if (RetVT.SimpleTy != MVT::i32)
16465
0
    return 0;
16466
0
  if ((Subtarget->hasAVX())) {
16467
0
    return fastEmitInst_rr(X86::VTESTPSYrr, &X86::VR256RegClass, Op0, Op1);
16468
0
  }
16469
0
  return 0;
16470
0
}
16471
16472
0
unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16473
0
  if (RetVT.SimpleTy != MVT::i32)
16474
0
    return 0;
16475
0
  if ((Subtarget->hasAVX())) {
16476
0
    return fastEmitInst_rr(X86::VTESTPDrr, &X86::VR128RegClass, Op0, Op1);
16477
0
  }
16478
0
  return 0;
16479
0
}
16480
16481
0
unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16482
0
  if (RetVT.SimpleTy != MVT::i32)
16483
0
    return 0;
16484
0
  if ((Subtarget->hasAVX())) {
16485
0
    return fastEmitInst_rr(X86::VTESTPDYrr, &X86::VR256RegClass, Op0, Op1);
16486
0
  }
16487
0
  return 0;
16488
0
}
16489
16490
0
unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16491
0
  switch (VT.SimpleTy) {
16492
0
  case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1);
16493
0
  case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1);
16494
0
  case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1);
16495
0
  case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1);
16496
0
  default: return 0;
16497
0
  }
16498
0
}
16499
16500
// FastEmit functions for X86ISD::UCOMI.
16501
16502
0
unsigned fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16503
0
  if (RetVT.SimpleTy != MVT::i32)
16504
0
    return 0;
16505
0
  if ((Subtarget->hasFP16())) {
16506
0
    return fastEmitInst_rr(X86::VUCOMISHZrr_Int, &X86::VR128XRegClass, Op0, Op1);
16507
0
  }
16508
0
  return 0;
16509
0
}
16510
16511
0
unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16512
0
  if (RetVT.SimpleTy != MVT::i32)
16513
0
    return 0;
16514
0
  if ((Subtarget->hasAVX512())) {
16515
0
    return fastEmitInst_rr(X86::VUCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op1);
16516
0
  }
16517
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16518
0
    return fastEmitInst_rr(X86::UCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1);
16519
0
  }
16520
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16521
0
    return fastEmitInst_rr(X86::VUCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1);
16522
0
  }
16523
0
  return 0;
16524
0
}
16525
16526
0
unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16527
0
  if (RetVT.SimpleTy != MVT::i32)
16528
0
    return 0;
16529
0
  if ((Subtarget->hasAVX512())) {
16530
0
    return fastEmitInst_rr(X86::VUCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op1);
16531
0
  }
16532
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16533
0
    return fastEmitInst_rr(X86::UCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1);
16534
0
  }
16535
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
16536
0
    return fastEmitInst_rr(X86::VUCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1);
16537
0
  }
16538
0
  return 0;
16539
0
}
16540
16541
0
unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16542
0
  switch (VT.SimpleTy) {
16543
0
  case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1);
16544
0
  case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1);
16545
0
  case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1);
16546
0
  default: return 0;
16547
0
  }
16548
0
}
16549
16550
// FastEmit functions for X86ISD::UNPCKH.
16551
16552
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16553
0
  if (RetVT.SimpleTy != MVT::v16i8)
16554
0
    return 0;
16555
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16556
0
    return fastEmitInst_rr(X86::VPUNPCKHBWZ128rr, &X86::VR128XRegClass, Op0, Op1);
16557
0
  }
16558
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16559
0
    return fastEmitInst_rr(X86::PUNPCKHBWrr, &X86::VR128RegClass, Op0, Op1);
16560
0
  }
16561
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16562
0
    return fastEmitInst_rr(X86::VPUNPCKHBWrr, &X86::VR128RegClass, Op0, Op1);
16563
0
  }
16564
0
  return 0;
16565
0
}
16566
16567
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16568
0
  if (RetVT.SimpleTy != MVT::v32i8)
16569
0
    return 0;
16570
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16571
0
    return fastEmitInst_rr(X86::VPUNPCKHBWZ256rr, &X86::VR256XRegClass, Op0, Op1);
16572
0
  }
16573
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16574
0
    return fastEmitInst_rr(X86::VPUNPCKHBWYrr, &X86::VR256RegClass, Op0, Op1);
16575
0
  }
16576
0
  return 0;
16577
0
}
16578
16579
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16580
0
  if (RetVT.SimpleTy != MVT::v64i8)
16581
0
    return 0;
16582
0
  if ((Subtarget->hasBWI())) {
16583
0
    return fastEmitInst_rr(X86::VPUNPCKHBWZrr, &X86::VR512RegClass, Op0, Op1);
16584
0
  }
16585
0
  return 0;
16586
0
}
16587
16588
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16589
0
  if (RetVT.SimpleTy != MVT::v8i16)
16590
0
    return 0;
16591
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16592
0
    return fastEmitInst_rr(X86::VPUNPCKHWDZ128rr, &X86::VR128XRegClass, Op0, Op1);
16593
0
  }
16594
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16595
0
    return fastEmitInst_rr(X86::PUNPCKHWDrr, &X86::VR128RegClass, Op0, Op1);
16596
0
  }
16597
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16598
0
    return fastEmitInst_rr(X86::VPUNPCKHWDrr, &X86::VR128RegClass, Op0, Op1);
16599
0
  }
16600
0
  return 0;
16601
0
}
16602
16603
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16604
0
  if (RetVT.SimpleTy != MVT::v16i16)
16605
0
    return 0;
16606
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16607
0
    return fastEmitInst_rr(X86::VPUNPCKHWDZ256rr, &X86::VR256XRegClass, Op0, Op1);
16608
0
  }
16609
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16610
0
    return fastEmitInst_rr(X86::VPUNPCKHWDYrr, &X86::VR256RegClass, Op0, Op1);
16611
0
  }
16612
0
  return 0;
16613
0
}
16614
16615
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16616
0
  if (RetVT.SimpleTy != MVT::v32i16)
16617
0
    return 0;
16618
0
  if ((Subtarget->hasBWI())) {
16619
0
    return fastEmitInst_rr(X86::VPUNPCKHWDZrr, &X86::VR512RegClass, Op0, Op1);
16620
0
  }
16621
0
  return 0;
16622
0
}
16623
16624
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16625
0
  if (RetVT.SimpleTy != MVT::v4i32)
16626
0
    return 0;
16627
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16628
0
    return fastEmitInst_rr(X86::VPUNPCKHDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
16629
0
  }
16630
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16631
0
    return fastEmitInst_rr(X86::PUNPCKHDQrr, &X86::VR128RegClass, Op0, Op1);
16632
0
  }
16633
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16634
0
    return fastEmitInst_rr(X86::VPUNPCKHDQrr, &X86::VR128RegClass, Op0, Op1);
16635
0
  }
16636
0
  return 0;
16637
0
}
16638
16639
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16640
0
  if (RetVT.SimpleTy != MVT::v8i32)
16641
0
    return 0;
16642
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16643
0
    return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op1);
16644
0
  }
16645
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16646
0
    return fastEmitInst_rr(X86::VPUNPCKHDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
16647
0
  }
16648
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16649
0
    return fastEmitInst_rr(X86::VPUNPCKHDQYrr, &X86::VR256RegClass, Op0, Op1);
16650
0
  }
16651
0
  return 0;
16652
0
}
16653
16654
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16655
0
  if (RetVT.SimpleTy != MVT::v16i32)
16656
0
    return 0;
16657
0
  if ((Subtarget->hasAVX512())) {
16658
0
    return fastEmitInst_rr(X86::VPUNPCKHDQZrr, &X86::VR512RegClass, Op0, Op1);
16659
0
  }
16660
0
  return 0;
16661
0
}
16662
16663
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16664
0
  if (RetVT.SimpleTy != MVT::v2i64)
16665
0
    return 0;
16666
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16667
0
    return fastEmitInst_rr(X86::VPUNPCKHQDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
16668
0
  }
16669
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16670
0
    return fastEmitInst_rr(X86::PUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op1);
16671
0
  }
16672
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16673
0
    return fastEmitInst_rr(X86::VPUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op1);
16674
0
  }
16675
0
  return 0;
16676
0
}
16677
16678
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16679
0
  if (RetVT.SimpleTy != MVT::v4i64)
16680
0
    return 0;
16681
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16682
0
    return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op1);
16683
0
  }
16684
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16685
0
    return fastEmitInst_rr(X86::VPUNPCKHQDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
16686
0
  }
16687
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16688
0
    return fastEmitInst_rr(X86::VPUNPCKHQDQYrr, &X86::VR256RegClass, Op0, Op1);
16689
0
  }
16690
0
  return 0;
16691
0
}
16692
16693
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16694
0
  if (RetVT.SimpleTy != MVT::v8i64)
16695
0
    return 0;
16696
0
  if ((Subtarget->hasAVX512())) {
16697
0
    return fastEmitInst_rr(X86::VPUNPCKHQDQZrr, &X86::VR512RegClass, Op0, Op1);
16698
0
  }
16699
0
  return 0;
16700
0
}
16701
16702
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16703
0
  if (RetVT.SimpleTy != MVT::v4f32)
16704
0
    return 0;
16705
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16706
0
    return fastEmitInst_rr(X86::VUNPCKHPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
16707
0
  }
16708
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16709
0
    return fastEmitInst_rr(X86::UNPCKHPSrr, &X86::VR128RegClass, Op0, Op1);
16710
0
  }
16711
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16712
0
    return fastEmitInst_rr(X86::VUNPCKHPSrr, &X86::VR128RegClass, Op0, Op1);
16713
0
  }
16714
0
  return 0;
16715
0
}
16716
16717
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16718
0
  if (RetVT.SimpleTy != MVT::v8f32)
16719
0
    return 0;
16720
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16721
0
    return fastEmitInst_rr(X86::VUNPCKHPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
16722
0
  }
16723
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16724
0
    return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op1);
16725
0
  }
16726
0
  return 0;
16727
0
}
16728
16729
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16730
0
  if (RetVT.SimpleTy != MVT::v16f32)
16731
0
    return 0;
16732
0
  if ((Subtarget->hasAVX512())) {
16733
0
    return fastEmitInst_rr(X86::VUNPCKHPSZrr, &X86::VR512RegClass, Op0, Op1);
16734
0
  }
16735
0
  return 0;
16736
0
}
16737
16738
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16739
0
  if (RetVT.SimpleTy != MVT::v2f64)
16740
0
    return 0;
16741
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16742
0
    return fastEmitInst_rr(X86::VUNPCKHPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
16743
0
  }
16744
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16745
0
    return fastEmitInst_rr(X86::UNPCKHPDrr, &X86::VR128RegClass, Op0, Op1);
16746
0
  }
16747
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16748
0
    return fastEmitInst_rr(X86::VUNPCKHPDrr, &X86::VR128RegClass, Op0, Op1);
16749
0
  }
16750
0
  return 0;
16751
0
}
16752
16753
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16754
0
  if (RetVT.SimpleTy != MVT::v4f64)
16755
0
    return 0;
16756
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16757
0
    return fastEmitInst_rr(X86::VUNPCKHPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
16758
0
  }
16759
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16760
0
    return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op1);
16761
0
  }
16762
0
  return 0;
16763
0
}
16764
16765
0
unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16766
0
  if (RetVT.SimpleTy != MVT::v8f64)
16767
0
    return 0;
16768
0
  if ((Subtarget->hasAVX512())) {
16769
0
    return fastEmitInst_rr(X86::VUNPCKHPDZrr, &X86::VR512RegClass, Op0, Op1);
16770
0
  }
16771
0
  return 0;
16772
0
}
16773
16774
0
unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
16775
0
  switch (VT.SimpleTy) {
16776
0
  case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1);
16777
0
  case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1);
16778
0
  case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1);
16779
0
  case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1);
16780
0
  case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1);
16781
0
  case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1);
16782
0
  case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1);
16783
0
  case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1);
16784
0
  case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1);
16785
0
  case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1);
16786
0
  case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1);
16787
0
  case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1);
16788
0
  case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1);
16789
0
  case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1);
16790
0
  case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1);
16791
0
  case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1);
16792
0
  case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1);
16793
0
  case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1);
16794
0
  default: return 0;
16795
0
  }
16796
0
}
16797
16798
// FastEmit functions for X86ISD::UNPCKL.
16799
16800
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16801
0
  if (RetVT.SimpleTy != MVT::v16i8)
16802
0
    return 0;
16803
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16804
0
    return fastEmitInst_rr(X86::VPUNPCKLBWZ128rr, &X86::VR128XRegClass, Op0, Op1);
16805
0
  }
16806
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16807
0
    return fastEmitInst_rr(X86::PUNPCKLBWrr, &X86::VR128RegClass, Op0, Op1);
16808
0
  }
16809
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16810
0
    return fastEmitInst_rr(X86::VPUNPCKLBWrr, &X86::VR128RegClass, Op0, Op1);
16811
0
  }
16812
0
  return 0;
16813
0
}
16814
16815
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16816
0
  if (RetVT.SimpleTy != MVT::v32i8)
16817
0
    return 0;
16818
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16819
0
    return fastEmitInst_rr(X86::VPUNPCKLBWZ256rr, &X86::VR256XRegClass, Op0, Op1);
16820
0
  }
16821
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16822
0
    return fastEmitInst_rr(X86::VPUNPCKLBWYrr, &X86::VR256RegClass, Op0, Op1);
16823
0
  }
16824
0
  return 0;
16825
0
}
16826
16827
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16828
0
  if (RetVT.SimpleTy != MVT::v64i8)
16829
0
    return 0;
16830
0
  if ((Subtarget->hasBWI())) {
16831
0
    return fastEmitInst_rr(X86::VPUNPCKLBWZrr, &X86::VR512RegClass, Op0, Op1);
16832
0
  }
16833
0
  return 0;
16834
0
}
16835
16836
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16837
0
  if (RetVT.SimpleTy != MVT::v8i16)
16838
0
    return 0;
16839
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16840
0
    return fastEmitInst_rr(X86::VPUNPCKLWDZ128rr, &X86::VR128XRegClass, Op0, Op1);
16841
0
  }
16842
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16843
0
    return fastEmitInst_rr(X86::PUNPCKLWDrr, &X86::VR128RegClass, Op0, Op1);
16844
0
  }
16845
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16846
0
    return fastEmitInst_rr(X86::VPUNPCKLWDrr, &X86::VR128RegClass, Op0, Op1);
16847
0
  }
16848
0
  return 0;
16849
0
}
16850
16851
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16852
0
  if (RetVT.SimpleTy != MVT::v16i16)
16853
0
    return 0;
16854
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
16855
0
    return fastEmitInst_rr(X86::VPUNPCKLWDZ256rr, &X86::VR256XRegClass, Op0, Op1);
16856
0
  }
16857
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
16858
0
    return fastEmitInst_rr(X86::VPUNPCKLWDYrr, &X86::VR256RegClass, Op0, Op1);
16859
0
  }
16860
0
  return 0;
16861
0
}
16862
16863
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16864
0
  if (RetVT.SimpleTy != MVT::v32i16)
16865
0
    return 0;
16866
0
  if ((Subtarget->hasBWI())) {
16867
0
    return fastEmitInst_rr(X86::VPUNPCKLWDZrr, &X86::VR512RegClass, Op0, Op1);
16868
0
  }
16869
0
  return 0;
16870
0
}
16871
16872
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16873
0
  if (RetVT.SimpleTy != MVT::v4i32)
16874
0
    return 0;
16875
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16876
0
    return fastEmitInst_rr(X86::VPUNPCKLDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
16877
0
  }
16878
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16879
0
    return fastEmitInst_rr(X86::PUNPCKLDQrr, &X86::VR128RegClass, Op0, Op1);
16880
0
  }
16881
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16882
0
    return fastEmitInst_rr(X86::VPUNPCKLDQrr, &X86::VR128RegClass, Op0, Op1);
16883
0
  }
16884
0
  return 0;
16885
0
}
16886
16887
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16888
0
  if (RetVT.SimpleTy != MVT::v8i32)
16889
0
    return 0;
16890
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16891
0
    return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op1);
16892
0
  }
16893
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16894
0
    return fastEmitInst_rr(X86::VPUNPCKLDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
16895
0
  }
16896
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16897
0
    return fastEmitInst_rr(X86::VPUNPCKLDQYrr, &X86::VR256RegClass, Op0, Op1);
16898
0
  }
16899
0
  return 0;
16900
0
}
16901
16902
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16903
0
  if (RetVT.SimpleTy != MVT::v16i32)
16904
0
    return 0;
16905
0
  if ((Subtarget->hasAVX512())) {
16906
0
    return fastEmitInst_rr(X86::VPUNPCKLDQZrr, &X86::VR512RegClass, Op0, Op1);
16907
0
  }
16908
0
  return 0;
16909
0
}
16910
16911
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16912
0
  if (RetVT.SimpleTy != MVT::v2i64)
16913
0
    return 0;
16914
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16915
0
    return fastEmitInst_rr(X86::VPUNPCKLQDQZ128rr, &X86::VR128XRegClass, Op0, Op1);
16916
0
  }
16917
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16918
0
    return fastEmitInst_rr(X86::PUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op1);
16919
0
  }
16920
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16921
0
    return fastEmitInst_rr(X86::VPUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op1);
16922
0
  }
16923
0
  return 0;
16924
0
}
16925
16926
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16927
0
  if (RetVT.SimpleTy != MVT::v4i64)
16928
0
    return 0;
16929
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
16930
0
    return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op1);
16931
0
  }
16932
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16933
0
    return fastEmitInst_rr(X86::VPUNPCKLQDQZ256rr, &X86::VR256XRegClass, Op0, Op1);
16934
0
  }
16935
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
16936
0
    return fastEmitInst_rr(X86::VPUNPCKLQDQYrr, &X86::VR256RegClass, Op0, Op1);
16937
0
  }
16938
0
  return 0;
16939
0
}
16940
16941
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16942
0
  if (RetVT.SimpleTy != MVT::v8i64)
16943
0
    return 0;
16944
0
  if ((Subtarget->hasAVX512())) {
16945
0
    return fastEmitInst_rr(X86::VPUNPCKLQDQZrr, &X86::VR512RegClass, Op0, Op1);
16946
0
  }
16947
0
  return 0;
16948
0
}
16949
16950
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16951
0
  if (RetVT.SimpleTy != MVT::v4f32)
16952
0
    return 0;
16953
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16954
0
    return fastEmitInst_rr(X86::VUNPCKLPSZ128rr, &X86::VR128XRegClass, Op0, Op1);
16955
0
  }
16956
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
16957
0
    return fastEmitInst_rr(X86::UNPCKLPSrr, &X86::VR128RegClass, Op0, Op1);
16958
0
  }
16959
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16960
0
    return fastEmitInst_rr(X86::VUNPCKLPSrr, &X86::VR128RegClass, Op0, Op1);
16961
0
  }
16962
0
  return 0;
16963
0
}
16964
16965
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16966
0
  if (RetVT.SimpleTy != MVT::v8f32)
16967
0
    return 0;
16968
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16969
0
    return fastEmitInst_rr(X86::VUNPCKLPSZ256rr, &X86::VR256XRegClass, Op0, Op1);
16970
0
  }
16971
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16972
0
    return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op1);
16973
0
  }
16974
0
  return 0;
16975
0
}
16976
16977
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16978
0
  if (RetVT.SimpleTy != MVT::v16f32)
16979
0
    return 0;
16980
0
  if ((Subtarget->hasAVX512())) {
16981
0
    return fastEmitInst_rr(X86::VUNPCKLPSZrr, &X86::VR512RegClass, Op0, Op1);
16982
0
  }
16983
0
  return 0;
16984
0
}
16985
16986
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
16987
0
  if (RetVT.SimpleTy != MVT::v2f64)
16988
0
    return 0;
16989
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
16990
0
    return fastEmitInst_rr(X86::VUNPCKLPDZ128rr, &X86::VR128XRegClass, Op0, Op1);
16991
0
  }
16992
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
16993
0
    return fastEmitInst_rr(X86::UNPCKLPDrr, &X86::VR128RegClass, Op0, Op1);
16994
0
  }
16995
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
16996
0
    return fastEmitInst_rr(X86::VUNPCKLPDrr, &X86::VR128RegClass, Op0, Op1);
16997
0
  }
16998
0
  return 0;
16999
0
}
17000
17001
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17002
0
  if (RetVT.SimpleTy != MVT::v4f64)
17003
0
    return 0;
17004
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17005
0
    return fastEmitInst_rr(X86::VUNPCKLPDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17006
0
  }
17007
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17008
0
    return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op1);
17009
0
  }
17010
0
  return 0;
17011
0
}
17012
17013
0
unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17014
0
  if (RetVT.SimpleTy != MVT::v8f64)
17015
0
    return 0;
17016
0
  if ((Subtarget->hasAVX512())) {
17017
0
    return fastEmitInst_rr(X86::VUNPCKLPDZrr, &X86::VR512RegClass, Op0, Op1);
17018
0
  }
17019
0
  return 0;
17020
0
}
17021
17022
0
unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17023
0
  switch (VT.SimpleTy) {
17024
0
  case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1);
17025
0
  case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1);
17026
0
  case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1);
17027
0
  case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1);
17028
0
  case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1);
17029
0
  case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1);
17030
0
  case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1);
17031
0
  case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1);
17032
0
  case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1);
17033
0
  case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1);
17034
0
  case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1);
17035
0
  case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1);
17036
0
  case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1);
17037
0
  case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1);
17038
0
  case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1);
17039
0
  case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1);
17040
0
  case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1);
17041
0
  case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1);
17042
0
  default: return 0;
17043
0
  }
17044
0
}
17045
17046
// FastEmit functions for X86ISD::VFCMULC.
17047
17048
0
unsigned fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17049
0
  if (RetVT.SimpleTy != MVT::v4f32)
17050
0
    return 0;
17051
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17052
0
    return fastEmitInst_rr(X86::VFCMULCPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
17053
0
  }
17054
0
  return 0;
17055
0
}
17056
17057
0
unsigned fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17058
0
  if (RetVT.SimpleTy != MVT::v8f32)
17059
0
    return 0;
17060
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17061
0
    return fastEmitInst_rr(X86::VFCMULCPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
17062
0
  }
17063
0
  return 0;
17064
0
}
17065
17066
0
unsigned fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17067
0
  if (RetVT.SimpleTy != MVT::v16f32)
17068
0
    return 0;
17069
0
  if ((Subtarget->hasFP16())) {
17070
0
    return fastEmitInst_rr(X86::VFCMULCPHZrr, &X86::VR512RegClass, Op0, Op1);
17071
0
  }
17072
0
  return 0;
17073
0
}
17074
17075
0
unsigned fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17076
0
  switch (VT.SimpleTy) {
17077
0
  case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
17078
0
  case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
17079
0
  case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
17080
0
  default: return 0;
17081
0
  }
17082
0
}
17083
17084
// FastEmit functions for X86ISD::VFCMULCSH.
17085
17086
0
unsigned fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17087
0
  if (RetVT.SimpleTy != MVT::v4f32)
17088
0
    return 0;
17089
0
  if ((Subtarget->hasFP16())) {
17090
0
    return fastEmitInst_rr(X86::VFCMULCSHZrr, &X86::VR128XRegClass, Op0, Op1);
17091
0
  }
17092
0
  return 0;
17093
0
}
17094
17095
0
unsigned fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17096
0
  switch (VT.SimpleTy) {
17097
0
  case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
17098
0
  default: return 0;
17099
0
  }
17100
0
}
17101
17102
// FastEmit functions for X86ISD::VFMULC.
17103
17104
0
unsigned fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17105
0
  if (RetVT.SimpleTy != MVT::v4f32)
17106
0
    return 0;
17107
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17108
0
    return fastEmitInst_rr(X86::VFMULCPHZ128rr, &X86::VR128XRegClass, Op0, Op1);
17109
0
  }
17110
0
  return 0;
17111
0
}
17112
17113
0
unsigned fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17114
0
  if (RetVT.SimpleTy != MVT::v8f32)
17115
0
    return 0;
17116
0
  if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) {
17117
0
    return fastEmitInst_rr(X86::VFMULCPHZ256rr, &X86::VR256XRegClass, Op0, Op1);
17118
0
  }
17119
0
  return 0;
17120
0
}
17121
17122
0
unsigned fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17123
0
  if (RetVT.SimpleTy != MVT::v16f32)
17124
0
    return 0;
17125
0
  if ((Subtarget->hasFP16())) {
17126
0
    return fastEmitInst_rr(X86::VFMULCPHZrr, &X86::VR512RegClass, Op0, Op1);
17127
0
  }
17128
0
  return 0;
17129
0
}
17130
17131
0
unsigned fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17132
0
  switch (VT.SimpleTy) {
17133
0
  case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1);
17134
0
  case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1);
17135
0
  case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1);
17136
0
  default: return 0;
17137
0
  }
17138
0
}
17139
17140
// FastEmit functions for X86ISD::VFMULCSH.
17141
17142
0
unsigned fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17143
0
  if (RetVT.SimpleTy != MVT::v4f32)
17144
0
    return 0;
17145
0
  if ((Subtarget->hasFP16())) {
17146
0
    return fastEmitInst_rr(X86::VFMULCSHZrr, &X86::VR128XRegClass, Op0, Op1);
17147
0
  }
17148
0
  return 0;
17149
0
}
17150
17151
0
unsigned fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17152
0
  switch (VT.SimpleTy) {
17153
0
  case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1);
17154
0
  default: return 0;
17155
0
  }
17156
0
}
17157
17158
// FastEmit functions for X86ISD::VP2INTERSECT.
17159
17160
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17161
0
  if (RetVT.SimpleTy != MVT::Untyped)
17162
0
    return 0;
17163
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17164
0
    return fastEmitInst_rr(X86::VP2INTERSECTDZ128rr, &X86::VK4PAIRRegClass, Op0, Op1);
17165
0
  }
17166
0
  return 0;
17167
0
}
17168
17169
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17170
0
  if (RetVT.SimpleTy != MVT::Untyped)
17171
0
    return 0;
17172
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17173
0
    return fastEmitInst_rr(X86::VP2INTERSECTDZ256rr, &X86::VK8PAIRRegClass, Op0, Op1);
17174
0
  }
17175
0
  return 0;
17176
0
}
17177
17178
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17179
0
  if (RetVT.SimpleTy != MVT::Untyped)
17180
0
    return 0;
17181
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
17182
0
    return fastEmitInst_rr(X86::VP2INTERSECTDZrr, &X86::VK16PAIRRegClass, Op0, Op1);
17183
0
  }
17184
0
  return 0;
17185
0
}
17186
17187
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17188
0
  if (RetVT.SimpleTy != MVT::Untyped)
17189
0
    return 0;
17190
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17191
0
    return fastEmitInst_rr(X86::VP2INTERSECTQZ128rr, &X86::VK2PAIRRegClass, Op0, Op1);
17192
0
  }
17193
0
  return 0;
17194
0
}
17195
17196
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17197
0
  if (RetVT.SimpleTy != MVT::Untyped)
17198
0
    return 0;
17199
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
17200
0
    return fastEmitInst_rr(X86::VP2INTERSECTQZ256rr, &X86::VK4PAIRRegClass, Op0, Op1);
17201
0
  }
17202
0
  return 0;
17203
0
}
17204
17205
0
unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17206
0
  if (RetVT.SimpleTy != MVT::Untyped)
17207
0
    return 0;
17208
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
17209
0
    return fastEmitInst_rr(X86::VP2INTERSECTQZrr, &X86::VK8PAIRRegClass, Op0, Op1);
17210
0
  }
17211
0
  return 0;
17212
0
}
17213
17214
0
unsigned fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17215
0
  switch (VT.SimpleTy) {
17216
0
  case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1);
17217
0
  case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1);
17218
0
  case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1);
17219
0
  case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1);
17220
0
  case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1);
17221
0
  case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1);
17222
0
  default: return 0;
17223
0
  }
17224
0
}
17225
17226
// FastEmit functions for X86ISD::VPERMV.
17227
17228
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17229
0
  if (RetVT.SimpleTy != MVT::v16i8)
17230
0
    return 0;
17231
0
  if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17232
0
    return fastEmitInst_rr(X86::VPERMBZ128rr, &X86::VR128XRegClass, Op0, Op1);
17233
0
  }
17234
0
  return 0;
17235
0
}
17236
17237
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17238
0
  if (RetVT.SimpleTy != MVT::v32i8)
17239
0
    return 0;
17240
0
  if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
17241
0
    return fastEmitInst_rr(X86::VPERMBZ256rr, &X86::VR256XRegClass, Op0, Op1);
17242
0
  }
17243
0
  return 0;
17244
0
}
17245
17246
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17247
0
  if (RetVT.SimpleTy != MVT::v64i8)
17248
0
    return 0;
17249
0
  if ((Subtarget->hasVBMI())) {
17250
0
    return fastEmitInst_rr(X86::VPERMBZrr, &X86::VR512RegClass, Op0, Op1);
17251
0
  }
17252
0
  return 0;
17253
0
}
17254
17255
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17256
0
  if (RetVT.SimpleTy != MVT::v8i16)
17257
0
    return 0;
17258
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17259
0
    return fastEmitInst_rr(X86::VPERMWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17260
0
  }
17261
0
  return 0;
17262
0
}
17263
17264
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17265
0
  if (RetVT.SimpleTy != MVT::v16i16)
17266
0
    return 0;
17267
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17268
0
    return fastEmitInst_rr(X86::VPERMWZ256rr, &X86::VR256XRegClass, Op0, Op1);
17269
0
  }
17270
0
  return 0;
17271
0
}
17272
17273
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17274
0
  if (RetVT.SimpleTy != MVT::v32i16)
17275
0
    return 0;
17276
0
  if ((Subtarget->hasBWI())) {
17277
0
    return fastEmitInst_rr(X86::VPERMWZrr, &X86::VR512RegClass, Op0, Op1);
17278
0
  }
17279
0
  return 0;
17280
0
}
17281
17282
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17283
0
  if (RetVT.SimpleTy != MVT::v8i32)
17284
0
    return 0;
17285
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17286
0
    return fastEmitInst_rr(X86::VPERMDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17287
0
  }
17288
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17289
0
    return fastEmitInst_rr(X86::VPERMDYrr, &X86::VR256RegClass, Op0, Op1);
17290
0
  }
17291
0
  return 0;
17292
0
}
17293
17294
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17295
0
  if (RetVT.SimpleTy != MVT::v16i32)
17296
0
    return 0;
17297
0
  if ((Subtarget->hasAVX512())) {
17298
0
    return fastEmitInst_rr(X86::VPERMDZrr, &X86::VR512RegClass, Op0, Op1);
17299
0
  }
17300
0
  return 0;
17301
0
}
17302
17303
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17304
0
  if (RetVT.SimpleTy != MVT::v4i64)
17305
0
    return 0;
17306
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17307
0
    return fastEmitInst_rr(X86::VPERMQZ256rr, &X86::VR256XRegClass, Op0, Op1);
17308
0
  }
17309
0
  return 0;
17310
0
}
17311
17312
0
unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17313
0
  if (RetVT.SimpleTy != MVT::v8i64)
17314
0
    return 0;
17315
0
  if ((Subtarget->hasAVX512())) {
17316
0
    return fastEmitInst_rr(X86::VPERMQZrr, &X86::VR512RegClass, Op0, Op1);
17317
0
  }
17318
0
  return 0;
17319
0
}
17320
17321
0
unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17322
0
  switch (VT.SimpleTy) {
17323
0
  case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1);
17324
0
  case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1);
17325
0
  case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1);
17326
0
  case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1);
17327
0
  case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1);
17328
0
  case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1);
17329
0
  case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1);
17330
0
  case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1);
17331
0
  case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1);
17332
0
  case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1);
17333
0
  default: return 0;
17334
0
  }
17335
0
}
17336
17337
// FastEmit functions for X86ISD::VPMADDUBSW.
17338
17339
0
unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17340
0
  if (RetVT.SimpleTy != MVT::v8i16)
17341
0
    return 0;
17342
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17343
0
    return fastEmitInst_rr(X86::VPMADDUBSWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17344
0
  }
17345
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
17346
0
    return fastEmitInst_rr(X86::PMADDUBSWrr, &X86::VR128RegClass, Op0, Op1);
17347
0
  }
17348
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17349
0
    return fastEmitInst_rr(X86::VPMADDUBSWrr, &X86::VR128RegClass, Op0, Op1);
17350
0
  }
17351
0
  return 0;
17352
0
}
17353
17354
0
unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17355
0
  if (RetVT.SimpleTy != MVT::v16i16)
17356
0
    return 0;
17357
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17358
0
    return fastEmitInst_rr(X86::VPMADDUBSWZ256rr, &X86::VR256XRegClass, Op0, Op1);
17359
0
  }
17360
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17361
0
    return fastEmitInst_rr(X86::VPMADDUBSWYrr, &X86::VR256RegClass, Op0, Op1);
17362
0
  }
17363
0
  return 0;
17364
0
}
17365
17366
0
unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17367
0
  if (RetVT.SimpleTy != MVT::v32i16)
17368
0
    return 0;
17369
0
  if ((Subtarget->hasBWI())) {
17370
0
    return fastEmitInst_rr(X86::VPMADDUBSWZrr, &X86::VR512RegClass, Op0, Op1);
17371
0
  }
17372
0
  return 0;
17373
0
}
17374
17375
0
unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17376
0
  switch (VT.SimpleTy) {
17377
0
  case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1);
17378
0
  case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1);
17379
0
  case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1);
17380
0
  default: return 0;
17381
0
  }
17382
0
}
17383
17384
// FastEmit functions for X86ISD::VPMADDWD.
17385
17386
0
unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17387
0
  if (RetVT.SimpleTy != MVT::v4i32)
17388
0
    return 0;
17389
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17390
0
    return fastEmitInst_rr(X86::VPMADDWDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17391
0
  }
17392
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17393
0
    return fastEmitInst_rr(X86::PMADDWDrr, &X86::VR128RegClass, Op0, Op1);
17394
0
  }
17395
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17396
0
    return fastEmitInst_rr(X86::VPMADDWDrr, &X86::VR128RegClass, Op0, Op1);
17397
0
  }
17398
0
  return 0;
17399
0
}
17400
17401
0
unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17402
0
  if (RetVT.SimpleTy != MVT::v8i32)
17403
0
    return 0;
17404
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17405
0
    return fastEmitInst_rr(X86::VPMADDWDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17406
0
  }
17407
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17408
0
    return fastEmitInst_rr(X86::VPMADDWDYrr, &X86::VR256RegClass, Op0, Op1);
17409
0
  }
17410
0
  return 0;
17411
0
}
17412
17413
0
unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17414
0
  if (RetVT.SimpleTy != MVT::v16i32)
17415
0
    return 0;
17416
0
  if ((Subtarget->hasBWI())) {
17417
0
    return fastEmitInst_rr(X86::VPMADDWDZrr, &X86::VR512RegClass, Op0, Op1);
17418
0
  }
17419
0
  return 0;
17420
0
}
17421
17422
0
unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17423
0
  switch (VT.SimpleTy) {
17424
0
  case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1);
17425
0
  case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1);
17426
0
  case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1);
17427
0
  default: return 0;
17428
0
  }
17429
0
}
17430
17431
// FastEmit functions for X86ISD::VPSHA.
17432
17433
0
unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17434
0
  if (RetVT.SimpleTy != MVT::v16i8)
17435
0
    return 0;
17436
0
  if ((Subtarget->hasXOP())) {
17437
0
    return fastEmitInst_rr(X86::VPSHABrr, &X86::VR128RegClass, Op0, Op1);
17438
0
  }
17439
0
  return 0;
17440
0
}
17441
17442
0
unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17443
0
  if (RetVT.SimpleTy != MVT::v8i16)
17444
0
    return 0;
17445
0
  if ((Subtarget->hasXOP())) {
17446
0
    return fastEmitInst_rr(X86::VPSHAWrr, &X86::VR128RegClass, Op0, Op1);
17447
0
  }
17448
0
  return 0;
17449
0
}
17450
17451
0
unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17452
0
  if (RetVT.SimpleTy != MVT::v4i32)
17453
0
    return 0;
17454
0
  if ((Subtarget->hasXOP())) {
17455
0
    return fastEmitInst_rr(X86::VPSHADrr, &X86::VR128RegClass, Op0, Op1);
17456
0
  }
17457
0
  return 0;
17458
0
}
17459
17460
0
unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17461
0
  if (RetVT.SimpleTy != MVT::v2i64)
17462
0
    return 0;
17463
0
  if ((Subtarget->hasXOP())) {
17464
0
    return fastEmitInst_rr(X86::VPSHAQrr, &X86::VR128RegClass, Op0, Op1);
17465
0
  }
17466
0
  return 0;
17467
0
}
17468
17469
0
unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17470
0
  switch (VT.SimpleTy) {
17471
0
  case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1);
17472
0
  case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1);
17473
0
  case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1);
17474
0
  case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1);
17475
0
  default: return 0;
17476
0
  }
17477
0
}
17478
17479
// FastEmit functions for X86ISD::VPSHL.
17480
17481
0
unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17482
0
  if (RetVT.SimpleTy != MVT::v16i8)
17483
0
    return 0;
17484
0
  if ((Subtarget->hasXOP())) {
17485
0
    return fastEmitInst_rr(X86::VPSHLBrr, &X86::VR128RegClass, Op0, Op1);
17486
0
  }
17487
0
  return 0;
17488
0
}
17489
17490
0
unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17491
0
  if (RetVT.SimpleTy != MVT::v8i16)
17492
0
    return 0;
17493
0
  if ((Subtarget->hasXOP())) {
17494
0
    return fastEmitInst_rr(X86::VPSHLWrr, &X86::VR128RegClass, Op0, Op1);
17495
0
  }
17496
0
  return 0;
17497
0
}
17498
17499
0
unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17500
0
  if (RetVT.SimpleTy != MVT::v4i32)
17501
0
    return 0;
17502
0
  if ((Subtarget->hasXOP())) {
17503
0
    return fastEmitInst_rr(X86::VPSHLDrr, &X86::VR128RegClass, Op0, Op1);
17504
0
  }
17505
0
  return 0;
17506
0
}
17507
17508
0
unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17509
0
  if (RetVT.SimpleTy != MVT::v2i64)
17510
0
    return 0;
17511
0
  if ((Subtarget->hasXOP())) {
17512
0
    return fastEmitInst_rr(X86::VPSHLQrr, &X86::VR128RegClass, Op0, Op1);
17513
0
  }
17514
0
  return 0;
17515
0
}
17516
17517
0
unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17518
0
  switch (VT.SimpleTy) {
17519
0
  case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1);
17520
0
  case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
17521
0
  case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
17522
0
  case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
17523
0
  default: return 0;
17524
0
  }
17525
0
}
17526
17527
// FastEmit functions for X86ISD::VPSHUFBITQMB.
17528
17529
0
unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17530
0
  if (RetVT.SimpleTy != MVT::v16i1)
17531
0
    return 0;
17532
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
17533
0
    return fastEmitInst_rr(X86::VPSHUFBITQMBZ128rr, &X86::VK16RegClass, Op0, Op1);
17534
0
  }
17535
0
  return 0;
17536
0
}
17537
17538
0
unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17539
0
  if (RetVT.SimpleTy != MVT::v32i1)
17540
0
    return 0;
17541
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
17542
0
    return fastEmitInst_rr(X86::VPSHUFBITQMBZ256rr, &X86::VK32RegClass, Op0, Op1);
17543
0
  }
17544
0
  return 0;
17545
0
}
17546
17547
0
unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17548
0
  if (RetVT.SimpleTy != MVT::v64i1)
17549
0
    return 0;
17550
0
  if ((Subtarget->hasBITALG())) {
17551
0
    return fastEmitInst_rr(X86::VPSHUFBITQMBZrr, &X86::VK64RegClass, Op0, Op1);
17552
0
  }
17553
0
  return 0;
17554
0
}
17555
17556
0
unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17557
0
  switch (VT.SimpleTy) {
17558
0
  case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1);
17559
0
  case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1);
17560
0
  case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1);
17561
0
  default: return 0;
17562
0
  }
17563
0
}
17564
17565
// FastEmit functions for X86ISD::VSHL.
17566
17567
0
unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17568
0
  if (RetVT.SimpleTy != MVT::v8i16)
17569
0
    return 0;
17570
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17571
0
    return fastEmitInst_rr(X86::VPSLLWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17572
0
  }
17573
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17574
0
    return fastEmitInst_rr(X86::PSLLWrr, &X86::VR128RegClass, Op0, Op1);
17575
0
  }
17576
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17577
0
    return fastEmitInst_rr(X86::VPSLLWrr, &X86::VR128RegClass, Op0, Op1);
17578
0
  }
17579
0
  return 0;
17580
0
}
17581
17582
0
unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17583
0
  if (RetVT.SimpleTy != MVT::v4i32)
17584
0
    return 0;
17585
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17586
0
    return fastEmitInst_rr(X86::VPSLLDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17587
0
  }
17588
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17589
0
    return fastEmitInst_rr(X86::PSLLDrr, &X86::VR128RegClass, Op0, Op1);
17590
0
  }
17591
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17592
0
    return fastEmitInst_rr(X86::VPSLLDrr, &X86::VR128RegClass, Op0, Op1);
17593
0
  }
17594
0
  return 0;
17595
0
}
17596
17597
0
unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17598
0
  if (RetVT.SimpleTy != MVT::v2i64)
17599
0
    return 0;
17600
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17601
0
    return fastEmitInst_rr(X86::VPSLLQZ128rr, &X86::VR128XRegClass, Op0, Op1);
17602
0
  }
17603
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17604
0
    return fastEmitInst_rr(X86::PSLLQrr, &X86::VR128RegClass, Op0, Op1);
17605
0
  }
17606
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17607
0
    return fastEmitInst_rr(X86::VPSLLQrr, &X86::VR128RegClass, Op0, Op1);
17608
0
  }
17609
0
  return 0;
17610
0
}
17611
17612
0
unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17613
0
  switch (VT.SimpleTy) {
17614
0
  case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1);
17615
0
  case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1);
17616
0
  case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1);
17617
0
  default: return 0;
17618
0
  }
17619
0
}
17620
17621
// FastEmit functions for X86ISD::VSHLV.
17622
17623
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17624
0
  if (RetVT.SimpleTy != MVT::v8i16)
17625
0
    return 0;
17626
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17627
0
    return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17628
0
  }
17629
0
  return 0;
17630
0
}
17631
17632
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17633
0
  if (RetVT.SimpleTy != MVT::v16i16)
17634
0
    return 0;
17635
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17636
0
    return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op1);
17637
0
  }
17638
0
  return 0;
17639
0
}
17640
17641
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17642
0
  if (RetVT.SimpleTy != MVT::v32i16)
17643
0
    return 0;
17644
0
  if ((Subtarget->hasBWI())) {
17645
0
    return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op1);
17646
0
  }
17647
0
  return 0;
17648
0
}
17649
17650
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17651
0
  if (RetVT.SimpleTy != MVT::v4i32)
17652
0
    return 0;
17653
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17654
0
    return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17655
0
  }
17656
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17657
0
    return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op1);
17658
0
  }
17659
0
  return 0;
17660
0
}
17661
17662
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17663
0
  if (RetVT.SimpleTy != MVT::v8i32)
17664
0
    return 0;
17665
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17666
0
    return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17667
0
  }
17668
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17669
0
    return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op1);
17670
0
  }
17671
0
  return 0;
17672
0
}
17673
17674
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17675
0
  if (RetVT.SimpleTy != MVT::v16i32)
17676
0
    return 0;
17677
0
  if ((Subtarget->hasAVX512())) {
17678
0
    return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op1);
17679
0
  }
17680
0
  return 0;
17681
0
}
17682
17683
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17684
0
  if (RetVT.SimpleTy != MVT::v2i64)
17685
0
    return 0;
17686
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17687
0
    return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op1);
17688
0
  }
17689
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17690
0
    return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op1);
17691
0
  }
17692
0
  return 0;
17693
0
}
17694
17695
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17696
0
  if (RetVT.SimpleTy != MVT::v4i64)
17697
0
    return 0;
17698
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17699
0
    return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op1);
17700
0
  }
17701
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17702
0
    return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op1);
17703
0
  }
17704
0
  return 0;
17705
0
}
17706
17707
0
unsigned fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17708
0
  if (RetVT.SimpleTy != MVT::v8i64)
17709
0
    return 0;
17710
0
  if ((Subtarget->hasAVX512())) {
17711
0
    return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op1);
17712
0
  }
17713
0
  return 0;
17714
0
}
17715
17716
0
unsigned fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17717
0
  switch (VT.SimpleTy) {
17718
0
  case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1);
17719
0
  case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1);
17720
0
  case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1);
17721
0
  case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1);
17722
0
  case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1);
17723
0
  case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1);
17724
0
  case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1);
17725
0
  case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1);
17726
0
  case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1);
17727
0
  default: return 0;
17728
0
  }
17729
0
}
17730
17731
// FastEmit functions for X86ISD::VSRA.
17732
17733
0
unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17734
0
  if (RetVT.SimpleTy != MVT::v8i16)
17735
0
    return 0;
17736
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17737
0
    return fastEmitInst_rr(X86::VPSRAWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17738
0
  }
17739
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17740
0
    return fastEmitInst_rr(X86::PSRAWrr, &X86::VR128RegClass, Op0, Op1);
17741
0
  }
17742
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17743
0
    return fastEmitInst_rr(X86::VPSRAWrr, &X86::VR128RegClass, Op0, Op1);
17744
0
  }
17745
0
  return 0;
17746
0
}
17747
17748
0
unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17749
0
  if (RetVT.SimpleTy != MVT::v4i32)
17750
0
    return 0;
17751
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17752
0
    return fastEmitInst_rr(X86::VPSRADZ128rr, &X86::VR128XRegClass, Op0, Op1);
17753
0
  }
17754
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17755
0
    return fastEmitInst_rr(X86::PSRADrr, &X86::VR128RegClass, Op0, Op1);
17756
0
  }
17757
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17758
0
    return fastEmitInst_rr(X86::VPSRADrr, &X86::VR128RegClass, Op0, Op1);
17759
0
  }
17760
0
  return 0;
17761
0
}
17762
17763
0
unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17764
0
  if (RetVT.SimpleTy != MVT::v2i64)
17765
0
    return 0;
17766
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17767
0
    return fastEmitInst_rr(X86::VPSRAQZ128rr, &X86::VR128XRegClass, Op0, Op1);
17768
0
  }
17769
0
  return 0;
17770
0
}
17771
17772
0
unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17773
0
  switch (VT.SimpleTy) {
17774
0
  case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1);
17775
0
  case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1);
17776
0
  case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1);
17777
0
  default: return 0;
17778
0
  }
17779
0
}
17780
17781
// FastEmit functions for X86ISD::VSRAV.
17782
17783
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17784
0
  if (RetVT.SimpleTy != MVT::v8i16)
17785
0
    return 0;
17786
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17787
0
    return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17788
0
  }
17789
0
  return 0;
17790
0
}
17791
17792
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17793
0
  if (RetVT.SimpleTy != MVT::v16i16)
17794
0
    return 0;
17795
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17796
0
    return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op1);
17797
0
  }
17798
0
  return 0;
17799
0
}
17800
17801
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17802
0
  if (RetVT.SimpleTy != MVT::v32i16)
17803
0
    return 0;
17804
0
  if ((Subtarget->hasBWI())) {
17805
0
    return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op1);
17806
0
  }
17807
0
  return 0;
17808
0
}
17809
17810
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17811
0
  if (RetVT.SimpleTy != MVT::v4i32)
17812
0
    return 0;
17813
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17814
0
    return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17815
0
  }
17816
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17817
0
    return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op1);
17818
0
  }
17819
0
  return 0;
17820
0
}
17821
17822
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17823
0
  if (RetVT.SimpleTy != MVT::v8i32)
17824
0
    return 0;
17825
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17826
0
    return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17827
0
  }
17828
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17829
0
    return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op1);
17830
0
  }
17831
0
  return 0;
17832
0
}
17833
17834
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17835
0
  if (RetVT.SimpleTy != MVT::v16i32)
17836
0
    return 0;
17837
0
  if ((Subtarget->hasAVX512())) {
17838
0
    return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op1);
17839
0
  }
17840
0
  return 0;
17841
0
}
17842
17843
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17844
0
  if (RetVT.SimpleTy != MVT::v2i64)
17845
0
    return 0;
17846
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17847
0
    return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op1);
17848
0
  }
17849
0
  return 0;
17850
0
}
17851
17852
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17853
0
  if (RetVT.SimpleTy != MVT::v4i64)
17854
0
    return 0;
17855
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17856
0
    return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op1);
17857
0
  }
17858
0
  return 0;
17859
0
}
17860
17861
0
unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17862
0
  if (RetVT.SimpleTy != MVT::v8i64)
17863
0
    return 0;
17864
0
  if ((Subtarget->hasAVX512())) {
17865
0
    return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op1);
17866
0
  }
17867
0
  return 0;
17868
0
}
17869
17870
0
unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17871
0
  switch (VT.SimpleTy) {
17872
0
  case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1);
17873
0
  case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1);
17874
0
  case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1);
17875
0
  case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1);
17876
0
  case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1);
17877
0
  case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1);
17878
0
  case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1);
17879
0
  case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1);
17880
0
  case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1);
17881
0
  default: return 0;
17882
0
  }
17883
0
}
17884
17885
// FastEmit functions for X86ISD::VSRL.
17886
17887
0
unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17888
0
  if (RetVT.SimpleTy != MVT::v8i16)
17889
0
    return 0;
17890
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17891
0
    return fastEmitInst_rr(X86::VPSRLWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17892
0
  }
17893
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17894
0
    return fastEmitInst_rr(X86::PSRLWrr, &X86::VR128RegClass, Op0, Op1);
17895
0
  }
17896
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
17897
0
    return fastEmitInst_rr(X86::VPSRLWrr, &X86::VR128RegClass, Op0, Op1);
17898
0
  }
17899
0
  return 0;
17900
0
}
17901
17902
0
unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17903
0
  if (RetVT.SimpleTy != MVT::v4i32)
17904
0
    return 0;
17905
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17906
0
    return fastEmitInst_rr(X86::VPSRLDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17907
0
  }
17908
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17909
0
    return fastEmitInst_rr(X86::PSRLDrr, &X86::VR128RegClass, Op0, Op1);
17910
0
  }
17911
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17912
0
    return fastEmitInst_rr(X86::VPSRLDrr, &X86::VR128RegClass, Op0, Op1);
17913
0
  }
17914
0
  return 0;
17915
0
}
17916
17917
0
unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17918
0
  if (RetVT.SimpleTy != MVT::v2i64)
17919
0
    return 0;
17920
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17921
0
    return fastEmitInst_rr(X86::VPSRLQZ128rr, &X86::VR128XRegClass, Op0, Op1);
17922
0
  }
17923
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
17924
0
    return fastEmitInst_rr(X86::PSRLQrr, &X86::VR128RegClass, Op0, Op1);
17925
0
  }
17926
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
17927
0
    return fastEmitInst_rr(X86::VPSRLQrr, &X86::VR128RegClass, Op0, Op1);
17928
0
  }
17929
0
  return 0;
17930
0
}
17931
17932
0
unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
17933
0
  switch (VT.SimpleTy) {
17934
0
  case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1);
17935
0
  case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1);
17936
0
  case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1);
17937
0
  default: return 0;
17938
0
  }
17939
0
}
17940
17941
// FastEmit functions for X86ISD::VSRLV.
17942
17943
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17944
0
  if (RetVT.SimpleTy != MVT::v8i16)
17945
0
    return 0;
17946
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17947
0
    return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op1);
17948
0
  }
17949
0
  return 0;
17950
0
}
17951
17952
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17953
0
  if (RetVT.SimpleTy != MVT::v16i16)
17954
0
    return 0;
17955
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
17956
0
    return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op1);
17957
0
  }
17958
0
  return 0;
17959
0
}
17960
17961
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17962
0
  if (RetVT.SimpleTy != MVT::v32i16)
17963
0
    return 0;
17964
0
  if ((Subtarget->hasBWI())) {
17965
0
    return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op1);
17966
0
  }
17967
0
  return 0;
17968
0
}
17969
17970
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17971
0
  if (RetVT.SimpleTy != MVT::v4i32)
17972
0
    return 0;
17973
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17974
0
    return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op1);
17975
0
  }
17976
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17977
0
    return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op1);
17978
0
  }
17979
0
  return 0;
17980
0
}
17981
17982
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17983
0
  if (RetVT.SimpleTy != MVT::v8i32)
17984
0
    return 0;
17985
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
17986
0
    return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op1);
17987
0
  }
17988
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
17989
0
    return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op1);
17990
0
  }
17991
0
  return 0;
17992
0
}
17993
17994
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
17995
0
  if (RetVT.SimpleTy != MVT::v16i32)
17996
0
    return 0;
17997
0
  if ((Subtarget->hasAVX512())) {
17998
0
    return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op1);
17999
0
  }
18000
0
  return 0;
18001
0
}
18002
18003
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
18004
0
  if (RetVT.SimpleTy != MVT::v2i64)
18005
0
    return 0;
18006
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18007
0
    return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op1);
18008
0
  }
18009
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18010
0
    return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op1);
18011
0
  }
18012
0
  return 0;
18013
0
}
18014
18015
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
18016
0
  if (RetVT.SimpleTy != MVT::v4i64)
18017
0
    return 0;
18018
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
18019
0
    return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op1);
18020
0
  }
18021
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
18022
0
    return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op1);
18023
0
  }
18024
0
  return 0;
18025
0
}
18026
18027
0
unsigned fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
18028
0
  if (RetVT.SimpleTy != MVT::v8i64)
18029
0
    return 0;
18030
0
  if ((Subtarget->hasAVX512())) {
18031
0
    return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op1);
18032
0
  }
18033
0
  return 0;
18034
0
}
18035
18036
0
unsigned fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
18037
0
  switch (VT.SimpleTy) {
18038
0
  case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1);
18039
0
  case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1);
18040
0
  case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1);
18041
0
  case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1);
18042
0
  case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1);
18043
0
  case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1);
18044
0
  case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1);
18045
0
  case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1);
18046
0
  case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1);
18047
0
  default: return 0;
18048
0
  }
18049
0
}
18050
18051
// Top-level FastEmit function.
18052
18053
0
unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override {
18054
0
  switch (Opcode) {
18055
0
  case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
18056
0
  case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
18057
0
  case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
18058
0
  case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
18059
0
  case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
18060
0
  case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
18061
0
  case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
18062
0
  case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
18063
0
  case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
18064
0
  case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
18065
0
  case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
18066
0
  case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1);
18067
0
  case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
18068
0
  case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
18069
0
  case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
18070
0
  case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
18071
0
  case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
18072
0
  case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
18073
0
  case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
18074
0
  case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
18075
0
  case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
18076
0
  case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
18077
0
  case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
18078
0
  case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
18079
0
  case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
18080
0
  case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
18081
0
  case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
18082
0
  case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
18083
0
  case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
18084
0
  case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
18085
0
  case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1);
18086
0
  case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1);
18087
0
  case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1);
18088
0
  case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1);
18089
0
  case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1);
18090
0
  case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1);
18091
0
  case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1);
18092
0
  case X86ISD::CVTNE2PS2BF16: return fastEmit_X86ISD_CVTNE2PS2BF16_rr(VT, RetVT, Op0, Op1);
18093
0
  case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1);
18094
0
  case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1);
18095
0
  case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1);
18096
0
  case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1);
18097
0
  case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1);
18098
0
  case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1);
18099
0
  case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1);
18100
0
  case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1);
18101
0
  case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1);
18102
0
  case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1);
18103
0
  case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1);
18104
0
  case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1);
18105
0
  case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1);
18106
0
  case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1);
18107
0
  case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1);
18108
0
  case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1);
18109
0
  case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1);
18110
0
  case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1);
18111
0
  case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1);
18112
0
  case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1);
18113
0
  case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1);
18114
0
  case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1);
18115
0
  case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1);
18116
0
  case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1);
18117
0
  case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1);
18118
0
  case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1);
18119
0
  case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1);
18120
0
  case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1);
18121
0
  case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1);
18122
0
  case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1);
18123
0
  case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1);
18124
0
  case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1);
18125
0
  case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1);
18126
0
  case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1);
18127
0
  case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1);
18128
0
  case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1);
18129
0
  case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1);
18130
0
  case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1);
18131
0
  case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1);
18132
0
  case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1);
18133
0
  case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1);
18134
0
  case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1);
18135
0
  case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op1);
18136
0
  case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op1);
18137
0
  case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1);
18138
0
  case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1);
18139
0
  case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1);
18140
0
  case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1);
18141
0
  case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1);
18142
0
  case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1);
18143
0
  case X86ISD::RCP28S: return fastEmit_X86ISD_RCP28S_rr(VT, RetVT, Op0, Op1);
18144
0
  case X86ISD::RCP28S_SAE: return fastEmit_X86ISD_RCP28S_SAE_rr(VT, RetVT, Op0, Op1);
18145
0
  case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1);
18146
0
  case X86ISD::RSQRT28S: return fastEmit_X86ISD_RSQRT28S_rr(VT, RetVT, Op0, Op1);
18147
0
  case X86ISD::RSQRT28S_SAE: return fastEmit_X86ISD_RSQRT28S_SAE_rr(VT, RetVT, Op0, Op1);
18148
0
  case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1);
18149
0
  case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1);
18150
0
  case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
18151
0
  case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1);
18152
0
  case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1);
18153
0
  case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1);
18154
0
  case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1);
18155
0
  case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1);
18156
0
  case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1);
18157
0
  case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1);
18158
0
  case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1);
18159
0
  case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1);
18160
0
  case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1);
18161
0
  case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1);
18162
0
  case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1);
18163
0
  case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1);
18164
0
  case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1);
18165
0
  case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1);
18166
0
  case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1);
18167
0
  case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1);
18168
0
  case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1);
18169
0
  case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1);
18170
0
  case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1);
18171
0
  case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1);
18172
0
  case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1);
18173
0
  case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1);
18174
0
  default: return 0;
18175
0
  }
18176
0
}
18177
18178
// FastEmit functions for ISD::ADD.
18179
18180
0
unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18181
0
  if (RetVT.SimpleTy != MVT::i8)
18182
0
    return 0;
18183
0
  if ((Subtarget->hasNDD())) {
18184
0
    return fastEmitInst_ri(X86::ADD8ri_ND, &X86::GR8RegClass, Op0, imm1);
18185
0
  }
18186
0
  if ((!Subtarget->hasNDD())) {
18187
0
    return fastEmitInst_ri(X86::ADD8ri, &X86::GR8RegClass, Op0, imm1);
18188
0
  }
18189
0
  return 0;
18190
0
}
18191
18192
0
unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18193
0
  if (RetVT.SimpleTy != MVT::i16)
18194
0
    return 0;
18195
0
  if ((Subtarget->hasNDD())) {
18196
0
    return fastEmitInst_ri(X86::ADD16ri_ND, &X86::GR16RegClass, Op0, imm1);
18197
0
  }
18198
0
  if ((!Subtarget->hasNDD())) {
18199
0
    return fastEmitInst_ri(X86::ADD16ri, &X86::GR16RegClass, Op0, imm1);
18200
0
  }
18201
0
  return 0;
18202
0
}
18203
18204
0
unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18205
0
  if (RetVT.SimpleTy != MVT::i32)
18206
0
    return 0;
18207
0
  if ((Subtarget->hasNDD())) {
18208
0
    return fastEmitInst_ri(X86::ADD32ri_ND, &X86::GR32RegClass, Op0, imm1);
18209
0
  }
18210
0
  if ((!Subtarget->hasNDD())) {
18211
0
    return fastEmitInst_ri(X86::ADD32ri, &X86::GR32RegClass, Op0, imm1);
18212
0
  }
18213
0
  return 0;
18214
0
}
18215
18216
0
unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18217
0
  switch (VT.SimpleTy) {
18218
0
  case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1);
18219
0
  case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1);
18220
0
  case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1);
18221
0
  default: return 0;
18222
0
  }
18223
0
}
18224
18225
// FastEmit functions for ISD::AND.
18226
18227
0
unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18228
0
  if (RetVT.SimpleTy != MVT::i8)
18229
0
    return 0;
18230
0
  if ((Subtarget->hasNDD())) {
18231
0
    return fastEmitInst_ri(X86::AND8ri_ND, &X86::GR8RegClass, Op0, imm1);
18232
0
  }
18233
0
  if ((!Subtarget->hasNDD())) {
18234
0
    return fastEmitInst_ri(X86::AND8ri, &X86::GR8RegClass, Op0, imm1);
18235
0
  }
18236
0
  return 0;
18237
0
}
18238
18239
0
unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18240
0
  if (RetVT.SimpleTy != MVT::i16)
18241
0
    return 0;
18242
0
  if ((Subtarget->hasNDD())) {
18243
0
    return fastEmitInst_ri(X86::AND16ri_ND, &X86::GR16RegClass, Op0, imm1);
18244
0
  }
18245
0
  if ((!Subtarget->hasNDD())) {
18246
0
    return fastEmitInst_ri(X86::AND16ri, &X86::GR16RegClass, Op0, imm1);
18247
0
  }
18248
0
  return 0;
18249
0
}
18250
18251
0
unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18252
0
  if (RetVT.SimpleTy != MVT::i32)
18253
0
    return 0;
18254
0
  if ((Subtarget->hasNDD())) {
18255
0
    return fastEmitInst_ri(X86::AND32ri_ND, &X86::GR32RegClass, Op0, imm1);
18256
0
  }
18257
0
  if ((!Subtarget->hasNDD())) {
18258
0
    return fastEmitInst_ri(X86::AND32ri, &X86::GR32RegClass, Op0, imm1);
18259
0
  }
18260
0
  return 0;
18261
0
}
18262
18263
0
unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18264
0
  switch (VT.SimpleTy) {
18265
0
  case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1);
18266
0
  case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1);
18267
0
  case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1);
18268
0
  default: return 0;
18269
0
  }
18270
0
}
18271
18272
// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
18273
18274
0
unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18275
0
  if (RetVT.SimpleTy != MVT::i32)
18276
0
    return 0;
18277
0
  if ((Subtarget->hasDQI())) {
18278
0
    return fastEmitInst_ri(X86::VPEXTRDZrr, &X86::GR32RegClass, Op0, imm1);
18279
0
  }
18280
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18281
0
    return fastEmitInst_ri(X86::PEXTRDrr, &X86::GR32RegClass, Op0, imm1);
18282
0
  }
18283
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
18284
0
    return fastEmitInst_ri(X86::VPEXTRDrr, &X86::GR32RegClass, Op0, imm1);
18285
0
  }
18286
0
  return 0;
18287
0
}
18288
18289
0
unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18290
0
  if (RetVT.SimpleTy != MVT::i64)
18291
0
    return 0;
18292
0
  if ((Subtarget->hasDQI())) {
18293
0
    return fastEmitInst_ri(X86::VPEXTRQZrr, &X86::GR64RegClass, Op0, imm1);
18294
0
  }
18295
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
18296
0
    return fastEmitInst_ri(X86::PEXTRQrr, &X86::GR64RegClass, Op0, imm1);
18297
0
  }
18298
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
18299
0
    return fastEmitInst_ri(X86::VPEXTRQrr, &X86::GR64RegClass, Op0, imm1);
18300
0
  }
18301
0
  return 0;
18302
0
}
18303
18304
0
unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18305
0
  switch (VT.SimpleTy) {
18306
0
  case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1);
18307
0
  case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1);
18308
0
  default: return 0;
18309
0
  }
18310
0
}
18311
18312
// FastEmit functions for ISD::MUL.
18313
18314
0
unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18315
0
  if (RetVT.SimpleTy != MVT::i16)
18316
0
    return 0;
18317
0
  return fastEmitInst_ri(X86::IMUL16rri, &X86::GR16RegClass, Op0, imm1);
18318
0
}
18319
18320
0
unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18321
0
  if (RetVT.SimpleTy != MVT::i32)
18322
0
    return 0;
18323
0
  return fastEmitInst_ri(X86::IMUL32rri, &X86::GR32RegClass, Op0, imm1);
18324
0
}
18325
18326
0
unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18327
0
  switch (VT.SimpleTy) {
18328
0
  case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1);
18329
0
  case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1);
18330
0
  default: return 0;
18331
0
  }
18332
0
}
18333
18334
// FastEmit functions for ISD::OR.
18335
18336
0
unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18337
0
  if (RetVT.SimpleTy != MVT::i8)
18338
0
    return 0;
18339
0
  if ((Subtarget->hasNDD())) {
18340
0
    return fastEmitInst_ri(X86::OR8ri_ND, &X86::GR8RegClass, Op0, imm1);
18341
0
  }
18342
0
  if ((!Subtarget->hasNDD())) {
18343
0
    return fastEmitInst_ri(X86::OR8ri, &X86::GR8RegClass, Op0, imm1);
18344
0
  }
18345
0
  return 0;
18346
0
}
18347
18348
0
unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18349
0
  if (RetVT.SimpleTy != MVT::i16)
18350
0
    return 0;
18351
0
  if ((Subtarget->hasNDD())) {
18352
0
    return fastEmitInst_ri(X86::OR16ri_ND, &X86::GR16RegClass, Op0, imm1);
18353
0
  }
18354
0
  if ((!Subtarget->hasNDD())) {
18355
0
    return fastEmitInst_ri(X86::OR16ri, &X86::GR16RegClass, Op0, imm1);
18356
0
  }
18357
0
  return 0;
18358
0
}
18359
18360
0
unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18361
0
  if (RetVT.SimpleTy != MVT::i32)
18362
0
    return 0;
18363
0
  if ((Subtarget->hasNDD())) {
18364
0
    return fastEmitInst_ri(X86::OR32ri_ND, &X86::GR32RegClass, Op0, imm1);
18365
0
  }
18366
0
  if ((!Subtarget->hasNDD())) {
18367
0
    return fastEmitInst_ri(X86::OR32ri, &X86::GR32RegClass, Op0, imm1);
18368
0
  }
18369
0
  return 0;
18370
0
}
18371
18372
0
unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18373
0
  switch (VT.SimpleTy) {
18374
0
  case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1);
18375
0
  case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1);
18376
0
  case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1);
18377
0
  default: return 0;
18378
0
  }
18379
0
}
18380
18381
// FastEmit functions for ISD::ROTL.
18382
18383
0
unsigned fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18384
0
  if (RetVT.SimpleTy != MVT::i8)
18385
0
    return 0;
18386
0
  return fastEmitInst_ri(X86::ROL8ri, &X86::GR8RegClass, Op0, imm1);
18387
0
}
18388
18389
0
unsigned fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18390
0
  if (RetVT.SimpleTy != MVT::i16)
18391
0
    return 0;
18392
0
  return fastEmitInst_ri(X86::ROL16ri, &X86::GR16RegClass, Op0, imm1);
18393
0
}
18394
18395
0
unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18396
0
  if (RetVT.SimpleTy != MVT::i32)
18397
0
    return 0;
18398
0
  if ((Subtarget->hasFastSHLDRotate())) {
18399
0
    return fastEmitInst_ri(X86::SHLDROT32ri, &X86::GR32RegClass, Op0, imm1);
18400
0
  }
18401
0
  return fastEmitInst_ri(X86::ROL32ri, &X86::GR32RegClass, Op0, imm1);
18402
0
}
18403
18404
0
unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18405
0
  if (RetVT.SimpleTy != MVT::i64)
18406
0
    return 0;
18407
0
  if ((Subtarget->hasFastSHLDRotate())) {
18408
0
    return fastEmitInst_ri(X86::SHLDROT64ri, &X86::GR64RegClass, Op0, imm1);
18409
0
  }
18410
0
  return fastEmitInst_ri(X86::ROL64ri, &X86::GR64RegClass, Op0, imm1);
18411
0
}
18412
18413
0
unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18414
0
  switch (VT.SimpleTy) {
18415
0
  case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1);
18416
0
  case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1);
18417
0
  case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1);
18418
0
  case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1);
18419
0
  default: return 0;
18420
0
  }
18421
0
}
18422
18423
// FastEmit functions for ISD::ROTR.
18424
18425
0
unsigned fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18426
0
  if (RetVT.SimpleTy != MVT::i8)
18427
0
    return 0;
18428
0
  return fastEmitInst_ri(X86::ROR8ri, &X86::GR8RegClass, Op0, imm1);
18429
0
}
18430
18431
0
unsigned fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18432
0
  if (RetVT.SimpleTy != MVT::i16)
18433
0
    return 0;
18434
0
  return fastEmitInst_ri(X86::ROR16ri, &X86::GR16RegClass, Op0, imm1);
18435
0
}
18436
18437
0
unsigned fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18438
0
  if (RetVT.SimpleTy != MVT::i32)
18439
0
    return 0;
18440
0
  if ((Subtarget->hasBMI2())) {
18441
0
    return fastEmitInst_ri(X86::RORX32ri, &X86::GR32RegClass, Op0, imm1);
18442
0
  }
18443
0
  if ((Subtarget->hasFastSHLDRotate())) {
18444
0
    return fastEmitInst_ri(X86::SHRDROT32ri, &X86::GR32RegClass, Op0, imm1);
18445
0
  }
18446
0
  return fastEmitInst_ri(X86::ROR32ri, &X86::GR32RegClass, Op0, imm1);
18447
0
}
18448
18449
0
unsigned fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18450
0
  if (RetVT.SimpleTy != MVT::i64)
18451
0
    return 0;
18452
0
  if ((Subtarget->hasBMI2())) {
18453
0
    return fastEmitInst_ri(X86::RORX64ri, &X86::GR64RegClass, Op0, imm1);
18454
0
  }
18455
0
  if ((Subtarget->hasFastSHLDRotate())) {
18456
0
    return fastEmitInst_ri(X86::SHRDROT64ri, &X86::GR64RegClass, Op0, imm1);
18457
0
  }
18458
0
  return fastEmitInst_ri(X86::ROR64ri, &X86::GR64RegClass, Op0, imm1);
18459
0
}
18460
18461
0
unsigned fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18462
0
  switch (VT.SimpleTy) {
18463
0
  case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1);
18464
0
  case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1);
18465
0
  case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1);
18466
0
  case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1);
18467
0
  default: return 0;
18468
0
  }
18469
0
}
18470
18471
// FastEmit functions for ISD::SHL.
18472
18473
0
unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18474
0
  if (RetVT.SimpleTy != MVT::i8)
18475
0
    return 0;
18476
0
  return fastEmitInst_ri(X86::SHL8ri, &X86::GR8RegClass, Op0, imm1);
18477
0
}
18478
18479
0
unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18480
0
  if (RetVT.SimpleTy != MVT::i16)
18481
0
    return 0;
18482
0
  return fastEmitInst_ri(X86::SHL16ri, &X86::GR16RegClass, Op0, imm1);
18483
0
}
18484
18485
0
unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18486
0
  if (RetVT.SimpleTy != MVT::i32)
18487
0
    return 0;
18488
0
  return fastEmitInst_ri(X86::SHL32ri, &X86::GR32RegClass, Op0, imm1);
18489
0
}
18490
18491
0
unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18492
0
  if (RetVT.SimpleTy != MVT::i64)
18493
0
    return 0;
18494
0
  return fastEmitInst_ri(X86::SHL64ri, &X86::GR64RegClass, Op0, imm1);
18495
0
}
18496
18497
0
unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18498
0
  switch (VT.SimpleTy) {
18499
0
  case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1);
18500
0
  case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1);
18501
0
  case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
18502
0
  case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1);
18503
0
  default: return 0;
18504
0
  }
18505
0
}
18506
18507
// FastEmit functions for ISD::SRA.
18508
18509
0
unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18510
0
  if (RetVT.SimpleTy != MVT::i8)
18511
0
    return 0;
18512
0
  return fastEmitInst_ri(X86::SAR8ri, &X86::GR8RegClass, Op0, imm1);
18513
0
}
18514
18515
0
unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18516
0
  if (RetVT.SimpleTy != MVT::i16)
18517
0
    return 0;
18518
0
  return fastEmitInst_ri(X86::SAR16ri, &X86::GR16RegClass, Op0, imm1);
18519
0
}
18520
18521
0
unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18522
0
  if (RetVT.SimpleTy != MVT::i32)
18523
0
    return 0;
18524
0
  return fastEmitInst_ri(X86::SAR32ri, &X86::GR32RegClass, Op0, imm1);
18525
0
}
18526
18527
0
unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18528
0
  if (RetVT.SimpleTy != MVT::i64)
18529
0
    return 0;
18530
0
  return fastEmitInst_ri(X86::SAR64ri, &X86::GR64RegClass, Op0, imm1);
18531
0
}
18532
18533
0
unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18534
0
  switch (VT.SimpleTy) {
18535
0
  case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1);
18536
0
  case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1);
18537
0
  case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1);
18538
0
  case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1);
18539
0
  default: return 0;
18540
0
  }
18541
0
}
18542
18543
// FastEmit functions for ISD::SRL.
18544
18545
0
unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18546
0
  if (RetVT.SimpleTy != MVT::i8)
18547
0
    return 0;
18548
0
  return fastEmitInst_ri(X86::SHR8ri, &X86::GR8RegClass, Op0, imm1);
18549
0
}
18550
18551
0
unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18552
0
  if (RetVT.SimpleTy != MVT::i16)
18553
0
    return 0;
18554
0
  return fastEmitInst_ri(X86::SHR16ri, &X86::GR16RegClass, Op0, imm1);
18555
0
}
18556
18557
0
unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18558
0
  if (RetVT.SimpleTy != MVT::i32)
18559
0
    return 0;
18560
0
  return fastEmitInst_ri(X86::SHR32ri, &X86::GR32RegClass, Op0, imm1);
18561
0
}
18562
18563
0
unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18564
0
  if (RetVT.SimpleTy != MVT::i64)
18565
0
    return 0;
18566
0
  return fastEmitInst_ri(X86::SHR64ri, &X86::GR64RegClass, Op0, imm1);
18567
0
}
18568
18569
0
unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18570
0
  switch (VT.SimpleTy) {
18571
0
  case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1);
18572
0
  case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1);
18573
0
  case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1);
18574
0
  case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1);
18575
0
  default: return 0;
18576
0
  }
18577
0
}
18578
18579
// FastEmit functions for ISD::SUB.
18580
18581
0
unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18582
0
  if (RetVT.SimpleTy != MVT::i8)
18583
0
    return 0;
18584
0
  if ((Subtarget->hasNDD())) {
18585
0
    return fastEmitInst_ri(X86::SUB8ri_ND, &X86::GR8RegClass, Op0, imm1);
18586
0
  }
18587
0
  if ((!Subtarget->hasNDD())) {
18588
0
    return fastEmitInst_ri(X86::SUB8ri, &X86::GR8RegClass, Op0, imm1);
18589
0
  }
18590
0
  return 0;
18591
0
}
18592
18593
0
unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18594
0
  if (RetVT.SimpleTy != MVT::i16)
18595
0
    return 0;
18596
0
  if ((Subtarget->hasNDD())) {
18597
0
    return fastEmitInst_ri(X86::SUB16ri_ND, &X86::GR16RegClass, Op0, imm1);
18598
0
  }
18599
0
  if ((!Subtarget->hasNDD())) {
18600
0
    return fastEmitInst_ri(X86::SUB16ri, &X86::GR16RegClass, Op0, imm1);
18601
0
  }
18602
0
  return 0;
18603
0
}
18604
18605
0
unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18606
0
  if (RetVT.SimpleTy != MVT::i32)
18607
0
    return 0;
18608
0
  if ((Subtarget->hasNDD())) {
18609
0
    return fastEmitInst_ri(X86::SUB32ri_ND, &X86::GR32RegClass, Op0, imm1);
18610
0
  }
18611
0
  if ((!Subtarget->hasNDD())) {
18612
0
    return fastEmitInst_ri(X86::SUB32ri, &X86::GR32RegClass, Op0, imm1);
18613
0
  }
18614
0
  return 0;
18615
0
}
18616
18617
0
unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18618
0
  switch (VT.SimpleTy) {
18619
0
  case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1);
18620
0
  case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1);
18621
0
  case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1);
18622
0
  default: return 0;
18623
0
  }
18624
0
}
18625
18626
// FastEmit functions for ISD::XOR.
18627
18628
0
unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18629
0
  if (RetVT.SimpleTy != MVT::i8)
18630
0
    return 0;
18631
0
  if ((Subtarget->hasNDD())) {
18632
0
    return fastEmitInst_ri(X86::XOR8ri_ND, &X86::GR8RegClass, Op0, imm1);
18633
0
  }
18634
0
  if ((!Subtarget->hasNDD())) {
18635
0
    return fastEmitInst_ri(X86::XOR8ri, &X86::GR8RegClass, Op0, imm1);
18636
0
  }
18637
0
  return 0;
18638
0
}
18639
18640
0
unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18641
0
  if (RetVT.SimpleTy != MVT::i16)
18642
0
    return 0;
18643
0
  if ((Subtarget->hasNDD())) {
18644
0
    return fastEmitInst_ri(X86::XOR16ri_ND, &X86::GR16RegClass, Op0, imm1);
18645
0
  }
18646
0
  if ((!Subtarget->hasNDD())) {
18647
0
    return fastEmitInst_ri(X86::XOR16ri, &X86::GR16RegClass, Op0, imm1);
18648
0
  }
18649
0
  return 0;
18650
0
}
18651
18652
0
unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18653
0
  if (RetVT.SimpleTy != MVT::i32)
18654
0
    return 0;
18655
0
  if ((Subtarget->hasNDD())) {
18656
0
    return fastEmitInst_ri(X86::XOR32ri_ND, &X86::GR32RegClass, Op0, imm1);
18657
0
  }
18658
0
  if ((!Subtarget->hasNDD())) {
18659
0
    return fastEmitInst_ri(X86::XOR32ri, &X86::GR32RegClass, Op0, imm1);
18660
0
  }
18661
0
  return 0;
18662
0
}
18663
18664
0
unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18665
0
  switch (VT.SimpleTy) {
18666
0
  case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1);
18667
0
  case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1);
18668
0
  case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1);
18669
0
  default: return 0;
18670
0
  }
18671
0
}
18672
18673
// FastEmit functions for X86ISD::BT.
18674
18675
0
unsigned fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18676
0
  if (RetVT.SimpleTy != MVT::i32)
18677
0
    return 0;
18678
0
  return fastEmitInst_ri(X86::BT16ri8, &X86::GR16RegClass, Op0, imm1);
18679
0
}
18680
18681
0
unsigned fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18682
0
  if (RetVT.SimpleTy != MVT::i32)
18683
0
    return 0;
18684
0
  return fastEmitInst_ri(X86::BT32ri8, &X86::GR32RegClass, Op0, imm1);
18685
0
}
18686
18687
0
unsigned fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
18688
0
  if (RetVT.SimpleTy != MVT::i32)
18689
0
    return 0;
18690
0
  return fastEmitInst_ri(X86::BT64ri8, &X86::GR64RegClass, Op0, imm1);
18691
0
}
18692
18693
0
unsigned fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18694
0
  switch (VT.SimpleTy) {
18695
0
  case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1);
18696
0
  case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1);
18697
0
  case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1);
18698
0
  default: return 0;
18699
0
  }
18700
0
}
18701
18702
// Top-level FastEmit function.
18703
18704
0
unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) override {
18705
0
  if (VT == MVT::i64 && Predicate_i64immSExt32(imm1))
18706
0
    if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1))
18707
0
      return Reg;
18708
18709
0
  switch (Opcode) {
18710
0
  case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1);
18711
0
  case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1);
18712
0
  case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
18713
0
  case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1);
18714
0
  case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1);
18715
0
  case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1);
18716
0
  case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1);
18717
0
  case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
18718
0
  case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1);
18719
0
  case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1);
18720
0
  case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1);
18721
0
  case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1);
18722
0
  case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1);
18723
0
  default: return 0;
18724
0
  }
18725
0
}
18726
18727
// FastEmit functions for ISD::ADD.
18728
18729
0
unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18730
0
  if (RetVT.SimpleTy != MVT::i64)
18731
0
    return 0;
18732
0
  if ((Subtarget->hasNDD())) {
18733
0
    return fastEmitInst_ri(X86::ADD64ri32_ND, &X86::GR64RegClass, Op0, imm1);
18734
0
  }
18735
0
  if ((!Subtarget->hasNDD())) {
18736
0
    return fastEmitInst_ri(X86::ADD64ri32, &X86::GR64RegClass, Op0, imm1);
18737
0
  }
18738
0
  return 0;
18739
0
}
18740
18741
0
unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18742
0
  switch (VT.SimpleTy) {
18743
0
  case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18744
0
  default: return 0;
18745
0
  }
18746
0
}
18747
18748
// FastEmit functions for ISD::AND.
18749
18750
0
unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18751
0
  if (RetVT.SimpleTy != MVT::i64)
18752
0
    return 0;
18753
0
  if ((Subtarget->hasNDD())) {
18754
0
    return fastEmitInst_ri(X86::AND64ri32_ND, &X86::GR64RegClass, Op0, imm1);
18755
0
  }
18756
0
  if ((!Subtarget->hasNDD())) {
18757
0
    return fastEmitInst_ri(X86::AND64ri32, &X86::GR64RegClass, Op0, imm1);
18758
0
  }
18759
0
  return 0;
18760
0
}
18761
18762
0
unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18763
0
  switch (VT.SimpleTy) {
18764
0
  case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18765
0
  default: return 0;
18766
0
  }
18767
0
}
18768
18769
// FastEmit functions for ISD::MUL.
18770
18771
0
unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18772
0
  if (RetVT.SimpleTy != MVT::i64)
18773
0
    return 0;
18774
0
  return fastEmitInst_ri(X86::IMUL64rri32, &X86::GR64RegClass, Op0, imm1);
18775
0
}
18776
18777
0
unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18778
0
  switch (VT.SimpleTy) {
18779
0
  case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18780
0
  default: return 0;
18781
0
  }
18782
0
}
18783
18784
// FastEmit functions for ISD::OR.
18785
18786
0
unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18787
0
  if (RetVT.SimpleTy != MVT::i64)
18788
0
    return 0;
18789
0
  if ((Subtarget->hasNDD())) {
18790
0
    return fastEmitInst_ri(X86::OR64ri32_ND, &X86::GR64RegClass, Op0, imm1);
18791
0
  }
18792
0
  if ((!Subtarget->hasNDD())) {
18793
0
    return fastEmitInst_ri(X86::OR64ri32, &X86::GR64RegClass, Op0, imm1);
18794
0
  }
18795
0
  return 0;
18796
0
}
18797
18798
0
unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18799
0
  switch (VT.SimpleTy) {
18800
0
  case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18801
0
  default: return 0;
18802
0
  }
18803
0
}
18804
18805
// FastEmit functions for ISD::SUB.
18806
18807
0
unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18808
0
  if (RetVT.SimpleTy != MVT::i64)
18809
0
    return 0;
18810
0
  if ((Subtarget->hasNDD())) {
18811
0
    return fastEmitInst_ri(X86::SUB64ri32_ND, &X86::GR64RegClass, Op0, imm1);
18812
0
  }
18813
0
  if ((!Subtarget->hasNDD())) {
18814
0
    return fastEmitInst_ri(X86::SUB64ri32, &X86::GR64RegClass, Op0, imm1);
18815
0
  }
18816
0
  return 0;
18817
0
}
18818
18819
0
unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18820
0
  switch (VT.SimpleTy) {
18821
0
  case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18822
0
  default: return 0;
18823
0
  }
18824
0
}
18825
18826
// FastEmit functions for ISD::XOR.
18827
18828
0
unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) {
18829
0
  if (RetVT.SimpleTy != MVT::i64)
18830
0
    return 0;
18831
0
  if ((Subtarget->hasNDD())) {
18832
0
    return fastEmitInst_ri(X86::XOR64ri32_ND, &X86::GR64RegClass, Op0, imm1);
18833
0
  }
18834
0
  if ((!Subtarget->hasNDD())) {
18835
0
    return fastEmitInst_ri(X86::XOR64ri32, &X86::GR64RegClass, Op0, imm1);
18836
0
  }
18837
0
  return 0;
18838
0
}
18839
18840
0
unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
18841
0
  switch (VT.SimpleTy) {
18842
0
  case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1);
18843
0
  default: return 0;
18844
0
  }
18845
0
}
18846
18847
// Top-level FastEmit function.
18848
18849
0
unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
18850
0
  switch (Opcode) {
18851
0
  case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18852
0
  case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18853
0
  case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18854
0
  case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18855
0
  case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18856
0
  case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1);
18857
0
  default: return 0;
18858
0
  }
18859
0
}
18860
18861
// FastEmit functions for ISD::Constant.
18862
18863
0
unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
18864
0
  if (RetVT.SimpleTy != MVT::i8)
18865
0
    return 0;
18866
0
  return fastEmitInst_i(X86::MOV8ri, &X86::GR8RegClass, imm0);
18867
0
}
18868
18869
0
unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
18870
0
  if (RetVT.SimpleTy != MVT::i16)
18871
0
    return 0;
18872
0
  return fastEmitInst_i(X86::MOV16ri, &X86::GR16RegClass, imm0);
18873
0
}
18874
18875
0
unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
18876
0
  if (RetVT.SimpleTy != MVT::i32)
18877
0
    return 0;
18878
0
  return fastEmitInst_i(X86::MOV32ri, &X86::GR32RegClass, imm0);
18879
0
}
18880
18881
0
unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
18882
0
  if (RetVT.SimpleTy != MVT::i64)
18883
0
    return 0;
18884
0
  return fastEmitInst_i(X86::MOV64ri, &X86::GR64RegClass, imm0);
18885
0
}
18886
18887
0
unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
18888
0
  switch (VT.SimpleTy) {
18889
0
  case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
18890
0
  case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
18891
0
  case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
18892
0
  case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
18893
0
  default: return 0;
18894
0
  }
18895
0
}
18896
18897
// Top-level FastEmit function.
18898
18899
0
unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
18900
0
  switch (Opcode) {
18901
0
  case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
18902
0
  default: return 0;
18903
0
  }
18904
0
}
18905